diff --git a/setup/dev/advanced.cfg b/setup/dev/advanced.cfg index 23981f0f34d..798221767fd 100644 --- a/setup/dev/advanced.cfg +++ b/setup/dev/advanced.cfg @@ -17,7 +17,7 @@ { "zones": [ { - "name": "Sandbox-simulator", + "name": "Sandbox-simulator-new", "guestcidraddress": "10.1.1.0/24", "dns1": "10.147.28.6", "physical_networks": [ @@ -137,7 +137,7 @@ "port": 3306, "user": "cloud" }, - "logger": + "logger": { "LogFolderPath": "/tmp/" }, @@ -218,5 +218,14 @@ "certCAPath": "NA", "certPath": "NA" } - ] + ], + "ApiLoadCfg": + { + "ParsedApiDestFolder": ".", + "ApiSpecFile": "/etc/cloud/cli/commands.xml" + }, + "TestData": + { + "Path": "config/config.cfg" + } } diff --git a/setup/dev/basic.cfg b/setup/dev/basic.cfg index 3e39d6dcb7b..196a191f6fc 100644 --- a/setup/dev/basic.cfg +++ b/setup/dev/basic.cfg @@ -172,5 +172,14 @@ "user": "root", "port": 8096 } - ] + ], + "ApiLoadCfg": + { + "ParsedApiDestFolder": ".", + "ApiSpecFile": "/etc/cloud/cli/commands.xml" + }, + "TestData": + { + "Path": "config/config.cfg" + } } diff --git a/test/integration/component/test_add_remove_network.py b/test/integration/component/test_add_remove_network.py index 4529ec7dfab..6eef71ea2d3 100644 --- a/test/integration/component/test_add_remove_network.py +++ b/test/integration/component/test_add_remove_network.py @@ -1300,7 +1300,7 @@ class TestFailureScenariosAddNetworkToVM(cloudstackTestCase): self.debug("Created account %s" % account.name) self.debug("creating user api client for account: %s" % account.name) - api_client = self.testClient.createUserApiClient(UserName=account.name, DomainName=self.account.domain) + api_client = self.testClient.getUserApiClient(UserName=account.name, DomainName=self.account.domain) self.debug("Trying to add network to vm with this api client, this should fail due to \ insufficient permission") @@ -1478,7 +1478,7 @@ class TestFailureScenariosRemoveNicFromVM(cloudstackTestCase): self.debug("Created account %s" % account.name) self.debug("creating user api client for account: %s" % account.name) - api_client = self.testClient.createUserApiClient(UserName=account.name, DomainName=self.account.domain) + api_client = self.testClient.getUserApiClient(UserName=account.name, DomainName=self.account.domain) self.debug("Trying to add network to vm with this api client, this should fail due to \ insufficient permission") @@ -1749,7 +1749,7 @@ class TestFailureScenariosUpdateVirtualMachineNIC(cloudstackTestCase): self.debug("Created account %s" % account.name) self.debug("creating user api client for account: %s" % account.name) - api_client = self.testClient.createUserApiClient(UserName=account.name, DomainName=self.account.domain) + api_client = self.testClient.getUserApiClient(UserName=account.name, DomainName=self.account.domain) self.debug("Listing virtual machine so that to retrive the list of non-default and default nic") vm_list = list_virtual_machines(self.apiclient, id=self.virtual_machine.id) diff --git a/test/integration/component/test_affinity_groups.py b/test/integration/component/test_affinity_groups.py index 7e4fabe5475..4fdb8f64f45 100644 --- a/test/integration/component/test_affinity_groups.py +++ b/test/integration/component/test_affinity_groups.py @@ -197,7 +197,7 @@ class TestCreateAffinityGroup(cloudstackTestCase): self.cleanup.append(self.do_admin) self.cleanup.append(self.new_domain) - domainapiclient = self.testClient.createUserApiClient(self.do_admin.name, self.new_domain.name, 2) + domainapiclient = self.testClient.getUserApiClient(self.do_admin.name, self.new_domain.name, 2) aff_grp = self.create_aff_grp(api_client=domainapiclient, aff_grp=self.services["host_anti_affinity"], acc=self.do_admin.name, domainid=self.new_domain.id) @@ -214,7 +214,7 @@ class TestCreateAffinityGroup(cloudstackTestCase): self.user = Account.create(self.api_client, self.services["new_account"], domainid=self.domain.id) - userapiclient = self.testClient.createUserApiClient(self.user.name, self.domain.name) + userapiclient = self.testClient.getUserApiClient(self.user.name, self.domain.name) self.cleanup.append(self.user) aff_grp = self.create_aff_grp(api_client=userapiclient, aff_grp=self.services["host_anti_affinity"], @@ -704,7 +704,7 @@ class TestDeleteAffinityGroups(cloudstackTestCase): self.user2 = Account.create(self.apiclient, self.services["new_account1"]) self.cleanup.append(self.user2) - userapiclient = self.testClient.createUserApiClient( + userapiclient = self.testClient.getUserApiClient( UserName=self.user2.name, DomainName=self.user2.domain, acctType=0) @@ -740,7 +740,7 @@ class TestDeleteAffinityGroups(cloudstackTestCase): self.user2 = Account.create(self.apiclient, self.services["new_account1"]) self.cleanup.append(self.user2) - userapiclient = self.testClient.createUserApiClient( + userapiclient = self.testClient.getUserApiClient( UserName=self.user2.name, DomainName=self.user2.domain, acctType=0) @@ -781,7 +781,7 @@ class TestDeleteAffinityGroups(cloudstackTestCase): self.services["new_account"]) self.cleanup.append(self.user1) - user1apiclient = self.testClient.createUserApiClient( + user1apiclient = self.testClient.getUserApiClient( UserName=self.user1.name, DomainName=self.user1.domain, acctType=0) @@ -1294,7 +1294,7 @@ class TestDeployVMAffinityGroups(cloudstackTestCase): self.user2 = Account.create(self.apiclient, self.services["new_account1"]) self.cleanup.append(self.user2) - userapiclient = self.testClient.createUserApiClient( + userapiclient = self.testClient.getUserApiClient( UserName=self.user2.name, DomainName=self.user2.domain, acctType=0) @@ -1327,7 +1327,7 @@ class TestDeployVMAffinityGroups(cloudstackTestCase): self.user2 = Account.create(self.apiclient, self.services["new_account1"]) self.cleanup.append(self.user2) - userapiclient = self.testClient.createUserApiClient( + userapiclient = self.testClient.getUserApiClient( UserName=self.user2.name, DomainName=self.user2.domain, acctType=0) @@ -1549,7 +1549,7 @@ class TestAffinityGroupsAdminUser(cloudstackTestCase): self.services["new_account"]) self.cleanup.append(self.user1) - userapiclient = self.testClient.createUserApiClient( + userapiclient = self.testClient.getUserApiClient( UserName=self.user1.name, DomainName=self.user1.domain, acctType=0) @@ -1588,7 +1588,7 @@ class TestAffinityGroupsAdminUser(cloudstackTestCase): self.services["new_account"]) self.cleanup.append(self.user1) - userapiclient = self.testClient.createUserApiClient( + userapiclient = self.testClient.getUserApiClient( UserName=self.user1.name, DomainName=self.user1.domain, acctType=0) @@ -1638,7 +1638,7 @@ class TestAffinityGroupsAdminUser(cloudstackTestCase): self.services["new_account"]) self.cleanup.append(self.user1) - userapiclient = self.testClient.createUserApiClient( + userapiclient = self.testClient.getUserApiClient( UserName=self.user1.name, DomainName=self.user1.domain, acctType=0) @@ -1674,7 +1674,7 @@ class TestAffinityGroupsAdminUser(cloudstackTestCase): self.services["new_account"]) self.cleanup.append(self.user1) - userapiclient = self.testClient.createUserApiClient( + userapiclient = self.testClient.getUserApiClient( UserName=self.user1.name, DomainName=self.user1.domain, acctType=0) @@ -1706,7 +1706,7 @@ class TestAffinityGroupsAdminUser(cloudstackTestCase): self.services["new_account"]) self.cleanup.append(self.user1) - userapiclient = self.testClient.createUserApiClient( + userapiclient = self.testClient.getUserApiClient( UserName=self.user1.name, DomainName=self.user1.domain, acctType=0) diff --git a/test/integration/component/test_cpu_domain_limits.py b/test/integration/component/test_cpu_domain_limits.py index c427e4fcf41..cd2ab541784 100644 --- a/test/integration/component/test_cpu_domain_limits.py +++ b/test/integration/component/test_cpu_domain_limits.py @@ -236,7 +236,7 @@ class TestDomainCPULimitsUpdateResources(cloudstackTestCase): self.account = admin self.domain = domain - api_client = self.testClient.createUserApiClient( + api_client = self.testClient.getUserApiClient( UserName=self.account.name, DomainName=self.account.domain) @@ -309,7 +309,7 @@ class TestDomainCPULimitsUpdateResources(cloudstackTestCase): self.account = admin self.domain = domain - api_client = self.testClient.createUserApiClient( + api_client = self.testClient.getUserApiClient( UserName=self.account.name, DomainName=self.account.domain) @@ -368,7 +368,7 @@ class TestDomainCPULimitsUpdateResources(cloudstackTestCase): self.account = admin self.domain = domain - api_client = self.testClient.createUserApiClient( + api_client = self.testClient.getUserApiClient( UserName=self.account.name, DomainName=self.account.domain) @@ -432,7 +432,7 @@ class TestDomainCPULimitsUpdateResources(cloudstackTestCase): self.account = admin self.domain = domain - api_client = self.testClient.createUserApiClient( + api_client = self.testClient.getUserApiClient( UserName=self.account.name, DomainName=self.account.domain) @@ -684,11 +684,11 @@ class TestMultipleChildDomains(cloudstackTestCase): self.debug("Setting up account and domain hierarchy") self.setupAccounts() - api_client_cadmin_1 = self.testClient.createUserApiClient( + api_client_cadmin_1 = self.testClient.getUserApiClient( UserName=self.cadmin_1.name, DomainName=self.cadmin_1.domain) - api_client_cadmin_2 = self.testClient.createUserApiClient( + api_client_cadmin_2 = self.testClient.getUserApiClient( UserName=self.cadmin_2.name, DomainName=self.cadmin_2.domain) diff --git a/test/integration/component/test_cpu_limits.py b/test/integration/component/test_cpu_limits.py index bdf2869c7c1..c6d8d1410f3 100644 --- a/test/integration/component/test_cpu_limits.py +++ b/test/integration/component/test_cpu_limits.py @@ -478,7 +478,7 @@ class TestDomainCPULimitsConfiguration(cloudstackTestCase): self.account = admin self.domain = domain - api_client = self.testClient.createUserApiClient( + api_client = self.testClient.getUserApiClient( UserName=self.account.name, DomainName=self.account.domain) @@ -550,7 +550,7 @@ class TestDomainCPULimitsConfiguration(cloudstackTestCase): self.account = admin self.domain = domain - api_client = self.testClient.createUserApiClient( + api_client = self.testClient.getUserApiClient( UserName=self.account.name, DomainName=self.account.domain) @@ -608,7 +608,7 @@ class TestDomainCPULimitsConfiguration(cloudstackTestCase): self.account = admin self.domain = domain - api_client = self.testClient.createUserApiClient( + api_client = self.testClient.getUserApiClient( UserName=self.account.name, DomainName=self.account.domain) @@ -680,7 +680,7 @@ class TestDomainCPULimitsConfiguration(cloudstackTestCase): if cpu_account_gc[0].max != 16: self.skipTest("This test case requires configuration value max.account.cpus to be 16") - api_client = self.testClient.createUserApiClient( + api_client = self.testClient.getUserApiClient( UserName=self.account.name, DomainName=self.account.domain) diff --git a/test/integration/component/test_cpu_max_limits.py b/test/integration/component/test_cpu_max_limits.py index 317df166a55..afbfcb40d01 100644 --- a/test/integration/component/test_cpu_max_limits.py +++ b/test/integration/component/test_cpu_max_limits.py @@ -251,7 +251,7 @@ class TestMaxCPULimits(cloudstackTestCase): self.debug("Setting up account and domain hierarchy") self.setupAccounts(account_limit=4, domain_limit=2) - api_client_admin = self.testClient.createUserApiClient( + api_client_admin = self.testClient.getUserApiClient( UserName=self.child_do_admin.name, DomainName=self.child_do_admin.domain) @@ -284,7 +284,7 @@ class TestMaxCPULimits(cloudstackTestCase): self.debug("Setting up account and domain hierarchy") self.setupAccounts(account_limit=6, domain_limit=8) - api_client_admin = self.testClient.createUserApiClient( + api_client_admin = self.testClient.getUserApiClient( UserName=self.child_do_admin.name, DomainName=self.child_do_admin.domain) @@ -325,7 +325,7 @@ class TestMaxCPULimits(cloudstackTestCase): self.debug("Setting up account and domain hierarchy") self.setupAccounts(account_limit=4, domain_limit=4, project_limit=2) - api_client_admin = self.testClient.createUserApiClient( + api_client_admin = self.testClient.getUserApiClient( UserName=self.child_do_admin.name, DomainName=self.child_do_admin.domain) @@ -360,7 +360,7 @@ class TestMaxCPULimits(cloudstackTestCase): self.debug("Setting up account and domain hierarchy") self.setupAccounts(account_limit=6, domain_limit=6, project_limit=6) - api_client_admin = self.testClient.createUserApiClient( + api_client_admin = self.testClient.getUserApiClient( UserName=self.child_do_admin.name, DomainName=self.child_do_admin.domain) diff --git a/test/integration/component/test_cpu_project_limits.py b/test/integration/component/test_cpu_project_limits.py index a8a1b3c3b24..a81c70ace10 100644 --- a/test/integration/component/test_cpu_project_limits.py +++ b/test/integration/component/test_cpu_project_limits.py @@ -136,7 +136,7 @@ class TestProjectsCPULimits(cloudstackTestCase): self.debug("Setting up account and domain hierarchy") self.setupProjectAccounts() - api_client = self.testClient.createUserApiClient( + api_client = self.testClient.getUserApiClient( UserName=self.admin.name, DomainName=self.admin.domain) diff --git a/test/integration/component/test_memory_limits.py b/test/integration/component/test_memory_limits.py index 7921e4be9ed..84af4b19a2d 100644 --- a/test/integration/component/test_memory_limits.py +++ b/test/integration/component/test_memory_limits.py @@ -495,7 +495,7 @@ class TestDomainMemoryLimitsConfiguration(cloudstackTestCase): self.account = admin self.domain = domain - api_client = self.testClient.createUserApiClient( + api_client = self.testClient.getUserApiClient( UserName=self.account.name, DomainName=self.account.domain) @@ -567,7 +567,7 @@ class TestDomainMemoryLimitsConfiguration(cloudstackTestCase): self.account = admin self.domain = domain - api_client = self.testClient.createUserApiClient( + api_client = self.testClient.getUserApiClient( UserName=self.account.name, DomainName=self.account.domain) @@ -625,7 +625,7 @@ class TestDomainMemoryLimitsConfiguration(cloudstackTestCase): self.account = admin self.domain = domain - api_client = self.testClient.createUserApiClient( + api_client = self.testClient.getUserApiClient( UserName=self.account.name, DomainName=self.account.domain) @@ -698,7 +698,7 @@ class TestDomainMemoryLimitsConfiguration(cloudstackTestCase): if memory_account_gc[0].max != 8192: self.skipTest("This test case requires configuration value max.account.memory to be 8192") - api_client = self.testClient.createUserApiClient( + api_client = self.testClient.getUserApiClient( UserName=self.account.name, DomainName=self.account.domain) diff --git a/test/integration/component/test_mm_domain_limits.py b/test/integration/component/test_mm_domain_limits.py index 68660c13cf5..a6c4de80f96 100644 --- a/test/integration/component/test_mm_domain_limits.py +++ b/test/integration/component/test_mm_domain_limits.py @@ -233,7 +233,7 @@ class TestDomainMemoryLimits(cloudstackTestCase): self.debug("Creating an instance with service offering: %s" % self.service_offering.name) - api_client = self.testClient.createUserApiClient( + api_client = self.testClient.getUserApiClient( UserName=self.account.name, DomainName=self.account.domain) @@ -370,7 +370,7 @@ class TestDomainMemoryLimits(cloudstackTestCase): self.debug("Creating an instance with service offering: %s" % self.service_offering.name) - api_client = self.testClient.createUserApiClient( + api_client = self.testClient.getUserApiClient( UserName=self.account.name, DomainName=self.account.domain) @@ -429,7 +429,7 @@ class TestDomainMemoryLimits(cloudstackTestCase): self.debug("Creating an instance with service offering: %s" % self.service_offering.name) - api_client = self.testClient.createUserApiClient( + api_client = self.testClient.getUserApiClient( UserName=self.account.name, DomainName=self.account.domain) @@ -489,7 +489,7 @@ class TestDomainMemoryLimits(cloudstackTestCase): self.debug("Creating an instance with service offering: %s" % self.service_offering.name) - api_client = self.testClient.createUserApiClient( + api_client = self.testClient.getUserApiClient( UserName=self.account.name, DomainName=self.account.domain) @@ -706,11 +706,11 @@ class TestMultipleChildDomainsMemory(cloudstackTestCase): self.debug("Setting up account and domain hierarchy") self.setupAccounts() - api_client_cadmin_1 = self.testClient.createUserApiClient( + api_client_cadmin_1 = self.testClient.getUserApiClient( UserName=self.cadmin_1.name, DomainName=self.cadmin_1.domain) - api_client_cadmin_2 = self.testClient.createUserApiClient( + api_client_cadmin_2 = self.testClient.getUserApiClient( UserName=self.cadmin_2.name, DomainName=self.cadmin_2.domain) diff --git a/test/integration/component/test_mm_max_limits.py b/test/integration/component/test_mm_max_limits.py index e10c119f07b..df29ea9fd23 100644 --- a/test/integration/component/test_mm_max_limits.py +++ b/test/integration/component/test_mm_max_limits.py @@ -253,7 +253,7 @@ class TestMaxMemoryLimits(cloudstackTestCase): self.debug("Setting up account and domain hierarchy") self.setupAccounts(account_limit=8, domain_limit=4) - api_client = self.testClient.createUserApiClient( + api_client = self.testClient.getUserApiClient( UserName=self.child_do_admin.name, DomainName=self.child_do_admin.domain) @@ -280,7 +280,7 @@ class TestMaxMemoryLimits(cloudstackTestCase): self.debug("Setting up account and domain hierarchy") self.setupAccounts(account_limit=7, domain_limit=14) - api_client = self.testClient.createUserApiClient( + api_client = self.testClient.getUserApiClient( UserName=self.child_do_admin.name, DomainName=self.child_do_admin.domain) @@ -310,7 +310,7 @@ class TestMaxMemoryLimits(cloudstackTestCase): self.debug("Setting up account and domain hierarchy") self.setupAccounts(account_limit=8,domain_limit=8, project_limit=4) - api_client = self.testClient.createUserApiClient( + api_client = self.testClient.getUserApiClient( UserName=self.child_do_admin.name, DomainName=self.child_do_admin.domain) @@ -334,7 +334,7 @@ class TestMaxMemoryLimits(cloudstackTestCase): self.debug("Setting up account and domain hierarchy") self.setupAccounts(account_limit=6, project_limit=12, domain_limit=12) - api_client = self.testClient.createUserApiClient( + api_client = self.testClient.getUserApiClient( UserName=self.child_do_admin.name, DomainName=self.child_do_admin.domain) diff --git a/test/integration/component/test_mm_project_limits.py b/test/integration/component/test_mm_project_limits.py index c314011090c..d030692f66e 100644 --- a/test/integration/component/test_mm_project_limits.py +++ b/test/integration/component/test_mm_project_limits.py @@ -136,7 +136,7 @@ class TestProjectsMemoryLimits(cloudstackTestCase): self.debug("Setting up account and domain hierarchy") self.setupProjectAccounts() - api_client = self.testClient.createUserApiClient( + api_client = self.testClient.getUserApiClient( UserName=self.admin.name, DomainName=self.admin.domain) diff --git a/test/integration/component/test_portable_ip.py b/test/integration/component/test_portable_ip.py index d397ec40312..8c652144757 100644 --- a/test/integration/component/test_portable_ip.py +++ b/test/integration/component/test_portable_ip.py @@ -221,7 +221,7 @@ class TestCreatePortablePublicIpRanges(cloudstackTestCase): domainid=self.domain.id ) - self.api_client_user = self.testClient.createUserApiClient( + self.api_client_user = self.testClient.getUserApiClient( UserName=self.account.name, DomainName=self.account.domain ) @@ -361,7 +361,7 @@ class TestDeletePortablePublicIpRanges(cloudstackTestCase): self.cleanup.append(self.account) - self.api_client_user = self.testClient.createUserApiClient( + self.api_client_user = self.testClient.getUserApiClient( UserName=self.account.name, DomainName=self.account.domain ) @@ -552,7 +552,7 @@ class TestListPortablePublicIpRanges(cloudstackTestCase): self.cleanup.append(self.account) - self.api_client_user = self.testClient.createUserApiClient( + self.api_client_user = self.testClient.getUserApiClient( UserName=self.account.name, DomainName=self.account.domain ) @@ -1110,7 +1110,7 @@ class TestDisassociatePublicIp(cloudstackTestCase): domainid=self.domain.id ) - self.api_client_user = self.testClient.createUserApiClient( + self.api_client_user = self.testClient.getUserApiClient( UserName=self.user_account.name, DomainName=self.user_account.domain ) diff --git a/test/integration/component/test_vpc.py b/test/integration/component/test_vpc.py index 1af8d8122ed..ab3b5abbe17 100644 --- a/test/integration/component/test_vpc.py +++ b/test/integration/component/test_vpc.py @@ -1934,7 +1934,7 @@ class TestVPC(cloudstackTestCase): self.debug("creating a VPC network in the account: %s" % user.name) - userapiclient = self.testClient.createUserApiClient( + userapiclient = self.testClient.getUserApiClient( UserName=user.name, DomainName=user.domain, acctType=0) diff --git a/test/integration/component/test_vpn_users.py b/test/integration/component/test_vpn_users.py index 02dd02622a3..d76cbf873e2 100644 --- a/test/integration/component/test_vpn_users.py +++ b/test/integration/component/test_vpn_users.py @@ -396,7 +396,7 @@ class TestVPNUsers(cloudstackTestCase): domainid=self.account.domainid) self.cleanup.append(admin) self.debug("Creating API client for newly created user") - api_client = self.testClient.createUserApiClient( + api_client = self.testClient.getUserApiClient( UserName=self.account.name, DomainName=self.account.domain) @@ -438,7 +438,7 @@ class TestVPNUsers(cloudstackTestCase): domainid=self.account.domainid) self.cleanup.append(admin) self.debug("Creating API client for newly created user") - api_client = self.testClient.createUserApiClient( + api_client = self.testClient.getUserApiClient( UserName=self.account.name, DomainName=self.account.domain) diff --git a/test/integration/smoke/test_affinity_groups.py b/test/integration/smoke/test_affinity_groups.py index c96a580f199..0f8bdc4edb1 100644 --- a/test/integration/smoke/test_affinity_groups.py +++ b/test/integration/smoke/test_affinity_groups.py @@ -70,7 +70,9 @@ class TestDeployVmWithAffinityGroup(cloudstackTestCase): @classmethod def setUpClass(cls): - cls.api_client = super(TestDeployVmWithAffinityGroup, cls).getClsTestClient().getApiClient() + cls.test_client = super(TestDeployVmWithAffinityGroup, cls).getClsTestClient() + zone_name = cls.test_client.getZoneForTests() + cls.api_client = cls.test_client.getApiClient() cls.services = Services().services # Get Zone, Domain and templates cls.domain = get_domain(cls.api_client, cls.services) diff --git a/test/integration/smoke/test_deploy_vm.py b/test/integration/smoke/test_deploy_vm.py index 425aeb749d2..41043863c4f 100644 --- a/test/integration/smoke/test_deploy_vm.py +++ b/test/integration/smoke/test_deploy_vm.py @@ -153,4 +153,4 @@ class TestDeployVM(cloudstackTestCase): try: cleanup_resources(self.apiclient, self.cleanup) except Exception as e: - self.debug("Warning! Exception in tearDown: %s" % e) \ No newline at end of file + self.debug("Warning! Exception in tearDown: %s" % e) diff --git a/test/integration/smoke/test_deploy_vm_with_userdata.py b/test/integration/smoke/test_deploy_vm_with_userdata.py index e3788cfbc22..85e768ede34 100644 --- a/test/integration/smoke/test_deploy_vm_with_userdata.py +++ b/test/integration/smoke/test_deploy_vm_with_userdata.py @@ -92,7 +92,7 @@ class TestDeployVmWithUserData(cloudstackTestCase): user_data = ''.join(random.choice(string.ascii_uppercase + string.digits) for x in range(2500)) cls.services["virtual_machine"]["userdata"] = user_data - @attr(tags=["simulator", "devcloud", "basic", "advanced"]) + @attr(tags=["simulator", "devcloud", "basic", "advanced", "post"]) def test_deployvm_userdata_post(self): """Test userdata as POST, size > 2k """ diff --git a/tools/marvin/marvin/asyncJobMgr.py b/tools/marvin/marvin/asyncJobMgr.py index e24170e18ff..dab85ca39a7 100644 --- a/tools/marvin/marvin/asyncJobMgr.py +++ b/tools/marvin/marvin/asyncJobMgr.py @@ -62,7 +62,7 @@ class workThread(threading.Thread): try: self.lock.acquire() result = self.connection.poll(job.jobId, job.responsecls).jobresult - except cloudstackException.cloudstackAPIException, e: + except cloudstackException.CloudstackAPIException, e: result = str(e) finally: self.lock.release() @@ -102,7 +102,7 @@ class workThread(threading.Thread): except: pass jobstatus.status = True - except cloudstackException.cloudstackAPIException, e: + except cloudstackException.CloudstackAPIException, e: jobstatus.result = str(e) jobstatus.status = False except: diff --git a/tools/marvin/marvin/cloudstackConnection.py b/tools/marvin/marvin/cloudstackConnection.py index fb03e3be3ac..8413ac0ad31 100644 --- a/tools/marvin/marvin/cloudstackConnection.py +++ b/tools/marvin/marvin/cloudstackConnection.py @@ -21,21 +21,33 @@ import base64 import hmac import hashlib import time -import cloudstackException from cloudstackAPI import * import jsonHelper +from codes import ( + FAILED, + INVALID_RESPONSE, + INVALID_INPUT, + JOB_FAILED, + JOB_INPROGRESS, + JOB_CANCELLED, + JOB_SUCCEEDED +) from requests import ( ConnectionError, HTTPError, Timeout, RequestException - ) +) +from cloudstackException import GetDetailExceptionInfo -class cloudConnection(object): - - """ Connections to make API calls to the cloudstack management server - """ +class CSConnection(object): + ''' + @Desc: Connection Class to make API\Command calls to the + CloudStack Management Server + Sends the GET\POST requests to CS based upon the + information provided and retrieves the parsed response. + ''' def __init__(self, mgmtDet, asyncTimeout=3600, logger=None, path='client/api'): self.apiKey = mgmtDet.apiKey @@ -44,68 +56,79 @@ class cloudConnection(object): self.port = mgmtDet.port self.user = mgmtDet.user self.passwd = mgmtDet.passwd - self.certCAPath = mgmtDet.certCAPath - self.certPath = mgmtDet.certPath + if mgmtDet.certCAPath != "NA" and mgmtDet.certPath != "NA": + self.certPath = (mgmtDet.certCAPath, mgmtDet.certPath) + else: + self.certPath = () self.logger = logger self.path = path self.retries = 5 self.mgtDetails = mgmtDet - self.protocol = "http" self.asyncTimeout = asyncTimeout self.auth = True if self.port == 8096 or \ (self.apiKey is None and self.securityKey is None): self.auth = False - if mgmtDet.useHttps == "True": - self.protocol = "https" - self.baseurl = "%s://%s:%d/%s"\ + self.protocol = "https" if mgmtDet.useHttps == "True" else "http" + self.httpsFlag = True if self.protocol == "https" else False + self.baseUrl = "%s://%s:%d/%s"\ % (self.protocol, self.mgtSvr, self.port, self.path) def __copy__(self): - return cloudConnection(self.mgtDetails, - self.asyncTimeout, - self.logger, - self.path) + return CSConnection(self.mgtDetails, + self.asyncTimeout, + self.logger, + self.path) - def poll(self, jobid, response): + def __poll(self, jobid, response_cmd): + ''' + @Desc: polls for the completion of a given jobid + @param 1. jobid: Monitor the Jobid for CS + 2. response_cmd:response command for request cmd + @return: FAILED if jobid is cancelled,failed + Else return async_response + ''' + try: + cmd = queryAsyncJobResult.queryAsyncJobResultCmd() + cmd.jobid = jobid + timeout = self.asyncTimeout + + while timeout > 0: + async_response = self.\ + marvinRequest(cmd, response_type=response_cmd) + if async_response != FAILED: + job_status = async_response.jobstatus + if job_status in [JOB_FAILED, JOB_CANCELLED]: + self.logger.debug("=====JobId:%s Either " + "got Cancelled or Failed======" + % (str(jobid))) + return FAILED + if job_status == JOB_SUCCEEDED: + self.logger.debug("======JobId:%s Succeeded=====" + % (str(jobid))) + return async_response + time.sleep(5) + timeout -= 5 + self.logger.debug("JobId:%s is Still Processing, " + "Will TimeOut in:%s" % (str(jobid), + str(timeout))) + return FAILED + except Exception, e: + self.logger.exception("__poll: Exception Occurred :%s" % + GetDetailExceptionInfo(e)) + return FAILED + + def __sign(self, payload): """ - polls the completion of a given jobid - @param jobid: - @param response: - @return: - """ - cmd = queryAsyncJobResult.queryAsyncJobResultCmd() - cmd.jobid = jobid - timeout = self.asyncTimeout - - while timeout > 0: - asyncResonse = self.marvinRequest(cmd, response_type=response) - - if asyncResonse.jobstatus == 2: - raise cloudstackException.cloudstackAPIException( - "asyncquery", asyncResonse.jobresult) - elif asyncResonse.jobstatus == 1: - return asyncResonse - - time.sleep(5) - if self.logger is not None: - self.logger.debug("job: %s still processing," - "will timeout in %ds" % (jobid, timeout)) - timeout = timeout - 5 - - raise cloudstackException.cloudstackAPIException( - "asyncquery", "Async job timeout %s" % jobid) - - def sign(self, payload): - """ - signs a given request URL when the apiKey and secretKey are known - + @Name : __sign + @Desc:signs a given request URL when the apiKey and + secretKey are known @param payload: dict of GET params to be signed @return: the signature of the payload """ params = zip(payload.keys(), payload.values()) params.sort(key=lambda k: str.lower(k[0])) - hashStr = "&".join( + hash_str = "&".join( ["=".join( [str.lower(r[0]), str.lower( @@ -114,168 +137,209 @@ class cloudConnection(object): ) for r in params] ) signature = base64.encodestring(hmac.new( - self.securityKey, hashStr, hashlib.sha1).digest()).strip() - self.logger.debug("Computed Signature by Marvin: %s" % signature) + self.securityKey, hash_str, hashlib.sha1).digest()).strip() return signature - def request(self, command, auth=True, payload={}, method='GET'): + def __sendPostReqToCS(self, url, payload): + ''' + @Name : __sendPostReqToCS + @Desc : Sends the POST Request to CS + @Input : url: URL to send post req + payload:Payload information as part of request + ''' + try: + response = requests.post(url, + params=payload, + cert=self.certPath, + verify=self.httpsFlag) + return response + except Exception, e: + self.logger.\ + exception("__sendPostReqToCS : Exception " + "Occurred: %s" % GetDetailExceptionInfo(e)) + return FAILED + + def __sendGetReqToCS(self, url, payload): + ''' + @Name : __sendGetReqToCS + @Desc : Sends the GET Request to CS + @Input : url: URL to send post req + payload:Payload information as part of request + ''' + try: + response = requests.get(url, + params=payload, + cert=self.certPath, + verify=self.httpsFlag) + return response + except Exception, e: + self.logger.exception("__sendGetReqToCS : Exception Occurred: %s" % + GetDetailExceptionInfo(e)) + return FAILED + + def __sendCmdToCS(self, command, auth=True, payload={}, method='GET'): """ - Makes requests using auth or over integration port + @Name : __sendCmdToCS + @Desc : Makes requests to CS using the Inputs provided @param command: cloudstack API command name eg: deployVirtualMachineCommand @param auth: Authentication (apikey,secretKey) => True else False for integration.api.port @param payload: request data composed as a dictionary @param method: GET/POST via HTTP - @return: + @output: FAILED or else response from CS """ - payload["command"] = command - payload["response"] = "json" - - if auth: - payload["apiKey"] = self.apiKey - signature = self.sign(payload) - payload["signature"] = signature - try: - #https_flag : Signifies whether to verify connection over \ - #http or https, \ - #initialized to False, will be set to true if user provided https - #connection - https_flag = False - cert_path = () - if self.protocol == "https": - https_flag = True - if self.certCAPath != "NA" and self.certPath != "NA": - cert_path = (self.certCAPath, self.certPath) + payload["command"] = command + payload["response"] = "json" + + if auth: + payload["apiKey"] = self.apiKey + payload["signature"] = self.__sign(payload) #Verify whether protocol is "http", then call the request over http if self.protocol == "http": + self.logger.debug("Payload: %s" % str(payload)) if method == 'POST': - response = requests.post(self.baseurl, params=payload, - verify=https_flag) - else: - response = requests.get(self.baseurl, params=payload, - verify=https_flag) + self.logger.debug("=======Sending POST Cmd : %s=======" + % str(command)) + return self.__sendPostReqToCS(self.baseUrl, payload) + if method == "GET": + self.logger.debug("========Sending GET Cmd : %s=======" + % str(command)) + return self.__sendGetReqToCS(self.baseUrl, payload) else: - ''' - If protocol is https, then create the connection url with \ - user provided certificates \ - provided as part of cert - ''' - try: - if method == 'POST': - response = requests.post(self.baseurl, - params=payload, - cert=cert_path, - verify=https_flag) - else: - response = requests.get(self.baseurl, params=payload, - cert=cert_path, - verify=https_flag) - except Exception, e: - ''' - If an exception occurs with user provided CA certs, \ - then try with default certs, \ - we dont need to mention here the cert path - ''' - self.logger.debug("Creating CS connection over https \ - didnt worked with user provided certs \ - , so trying with no certs %s" % e) - if method == 'POST': - response = requests.post(self.baseurl, - params=payload, - verify=https_flag) - else: - response = requests.get(self.baseurl, - params=payload, - verify=https_flag) - except ConnectionError, c: - self.logger.debug("Connection refused. Reason: %s : %s" % - (self.baseurl, c)) - raise c - except HTTPError, h: - self.logger.debug("Http Error.Server returned error code: %s" % h) - raise h - except Timeout, t: - self.logger.debug("Connection timed out with %s" % t) - raise t - except RequestException, r: - self.logger.debug("RequestException from server %s" % r) - raise r + self.logger.exception("__sendCmdToCS: Invalid Protocol") + return FAILED except Exception, e: - self.logger.debug("Error returned by server %s" % r) - raise e - else: - return response + self.logger.exception("__sendCmdToCS: Exception:%s" % + GetDetailExceptionInfo(e)) + return FAILED - def sanitizeCommand(self, cmd): + def __sanitizeCmd(self, cmd): """ - Removes None values, Validates all required params are present + @Name : __sanitizeCmd + @Desc : Removes None values, Validates all required params are present @param cmd: Cmd object eg: createPhysicalNetwork - @return: + @Output: Returns command name, asynchronous or not , request payload + INVALID_INPUT if cmd is invalid """ - requests = {} - required = [] - for attribute in dir(cmd): - if not attribute.startswith('__'): - if attribute == "isAsync": - isAsync = getattr(cmd, attribute) - elif attribute == "required": - required = getattr(cmd, attribute) - else: - requests[attribute] = getattr(cmd, attribute) - - cmdname = cmd.__class__.__name__.replace("Cmd", "") - for requiredPara in required: - if requests[requiredPara] is None: - raise cloudstackException.cloudstackAPIException( - cmdname, "%s is required" % requiredPara) - for param, value in requests.items(): - if value is None: - requests.pop(param) - elif isinstance(value, list): - if len(value) == 0: - requests.pop(param) - else: - if not isinstance(value[0], dict): - requests[param] = ",".join(value) + try: + cmd_name = '' + payload = {} + required = [] + for attribute in dir(cmd): + if not attribute.startswith('__'): + if attribute == "isAsync": + isAsync = getattr(cmd, attribute) + elif attribute == "required": + required = getattr(cmd, attribute) else: - requests.pop(param) - i = 0 - for val in value: - for k, v in val.iteritems(): - requests["%s[%d].%s" % (param, i, k)] = v - i = i + 1 - return cmdname.strip(), isAsync, requests + payload[attribute] = getattr(cmd, attribute) + cmd_name = cmd.__class__.__name__.replace("Cmd", "") + for required_param in required: + if payload[required_param] is None: + self.logger.debug("CmdName: %s Parameter : %s is Required" + % (cmd_name, required_param)) + return INVALID_INPUT + for param, value in payload.items(): + if value is None: + payload.pop(param) + elif isinstance(value, list): + if len(value) == 0: + payload.pop(param) + else: + if not isinstance(value[0], dict): + payload[param] = ",".join(value) + else: + payload.pop(param) + i = 0 + for val in value: + for k, v in val.iteritems(): + payload["%s[%d].%s" % (param, i, k)] = v + i += 1 + + return cmd_name.strip(), isAsync, payload + except Exception, e: + self.logger.\ + exception("__sanitizeCmd: CmdName : " + "%s : Exception:%s" % (cmd_name, + GetDetailExceptionInfo(e))) + return FAILED + + def __parseAndGetResponse(self, cmd_response, response_cls, is_async): + ''' + @Name : __parseAndGetResponse + @Desc : Verifies the Response(from CS) and returns an + appropriate json parsed Response + @Output: + ''' + if cmd_response == FAILED: + return FAILED + try: + ret = jsonHelper.getResultObj(cmd_response.json(), response_cls) + except TypeError: + ret = jsonHelper.getResultObj(cmd_response.json, response_cls) + + ''' + If the response is asynchronous, poll and return response + else return response as it is + ''' + if is_async == "false": + self.logger.debug("Response : %s" % str(ret)) + return ret + else: + response = self.__poll(ret.jobid, response_cls) + self.logger.debug("Response : %s" % str(response)) + return response.jobresult if response != FAILED else FAILED def marvinRequest(self, cmd, response_type=None, method='GET', data=''): """ - Requester for marvin command objects + @Name : marvinRequest + @Desc: Handles Marvin Requests @param cmd: marvin's command from cloudstackAPI @param response_type: response type of the command in cmd @param method: HTTP GET/POST, defaults to GET - @return: + @return: Response received from CS + FAILED In case of Error\Exception """ - cmdname, isAsync, payload = self.sanitizeCommand(cmd) - self.logger.debug("sending %s request: %s %s" % (method, cmdname, - str(payload))) - response = self.request(cmdname, - self.auth, - payload=payload, - method=method) - if response is None: - return None - self.logger.debug("Request: %s Response: %s" % (response.url, - response.text)) try: - response = jsonHelper.getResultObj(response.json(), response_type) - except TypeError: - response = jsonHelper.getResultObj(response.json, response_type) + ''' + 1. Verify the Inputs Provided + ''' + if (cmd is None or cmd == '')or \ + (response_type is None or response_type == ''): + self.logger.exception("marvinRequest : Invalid Command Input") + return FAILED - if isAsync == "false": - return response - else: - asyncJobId = response.jobid - response = self.poll(asyncJobId, response_type) - return response.jobresult + ''' + 2. Sanitize the Command + ''' + if self.__sanitizeCmd(cmd) != INVALID_INPUT: + cmd_name, is_async, payload = self.__sanitizeCmd(cmd) + else: + self.logger.exception("marvinRequest : Cmd: " + "Sanitizing Command Failed") + return FAILED + + ''' + 3. Send Command to CS + ''' + cmd_response = self.__sendCmdToCS(cmd_name, + self.auth, + payload=payload, + method=method) + if cmd_response == FAILED: + return FAILED + + ''' + 4. Check if the Command Response received above is valid or Not. + If not return Invalid Response + ''' + return self.__parseAndGetResponse(cmd_response, + response_type, + is_async) + except Exception, e: + self.logger.exception("marvinRequest : CmdName: %s Exception: %s" % + (str(cmd), GetDetailExceptionInfo(e))) + return FAILED diff --git a/tools/marvin/marvin/cloudstackException.py b/tools/marvin/marvin/cloudstackException.py index 6200003bbc3..3da28b76ef3 100644 --- a/tools/marvin/marvin/cloudstackException.py +++ b/tools/marvin/marvin/cloudstackException.py @@ -15,8 +15,12 @@ # specific language governing permissions and limitations # under the License. +import sys +import traceback +from marvin.codes import (INVALID_INPUT, EXCEPTION_OCCURRED) -class cloudstackAPIException(Exception): + +class CloudstackAPIException(Exception): def __init__(self, cmd="", result=""): self.errorMsg = "Execute cmd: %s failed, due to: %s" % (cmd, result) @@ -46,3 +50,12 @@ class internalError(Exception): def __str__(self): return self.errorMsg + + +def GetDetailExceptionInfo(e): + if e is not None: + exc_type, exc_value, exc_traceback = sys.exc_info() + return str(repr(traceback.format_exception( + exc_type, exc_value, exc_traceback))) + else: + return EXCEPTION_OCCURRED diff --git a/tools/marvin/marvin/cloudstackTestClient.py b/tools/marvin/marvin/cloudstackTestClient.py index 4ac510b03ab..f0ba135d2ff 100644 --- a/tools/marvin/marvin/cloudstackTestClient.py +++ b/tools/marvin/marvin/cloudstackTestClient.py @@ -15,201 +15,406 @@ # specific language governing permissions and limitations # under the License. -import cloudstackConnection +from cloudstackConnection import CSConnection import asyncJobMgr -import dbConnection +from dbConnection import DbConnection from cloudstackAPI import * import random import string import hashlib +from codes import (FAILED, PASS, ADMIN, DOMAIN_ADMIN, + USER, SUCCESS, XEN_SERVER) from configGenerator import ConfigManager -from marvin.integration.lib.utils import random_gen +from marvin.integration.lib import utils +from cloudstackException import GetDetailExceptionInfo +from marvin.integration.lib.utils import (random_gen, + validateList) +from marvin.cloudstackAPI.cloudstackAPIClient import CloudStackAPIClient ''' -@Desc : CloudStackTestClient is encapsulated class for getting various \ - clients viz., apiclient,dbconnection etc +@Desc : CloudStackTestClient is encapsulated entity for creating and + getting various clients viz., apiclient, + user api client, dbconnection, test Data parsed + information etc @Input : mgmtDetails : Management Server Details dbSvrDetails: Database Server details of Management \ Server. Retrieved from configuration file. - asyncTimeout : - defaultWorkerThreads : - logger : provides logging facilities for this library + asyncTimeout : Timeout for Async queries + defaultWorkerThreads : Number of worker threads + logger : provides logging facilities for this library + zone : The zone on which test suites using this test client will run ''' -class cloudstackTestClient(object): - def __init__(self, mgmtDetails, - dbSvrDetails, asyncTimeout=3600, - defaultWorkerThreads=10, - logger=None): - self.mgmtDetails = mgmtDetails - self.connection = \ - cloudstackConnection.cloudConnection(self.mgmtDetails, - asyncTimeout, - logger) - self.apiClient =\ - cloudstackAPIClient.CloudStackAPIClient(self.connection) - self.dbConnection = None - if dbSvrDetails is not None: - self.createDbConnection(dbSvrDetails.dbSvr, dbSvrDetails.port, - dbSvrDetails.user, - dbSvrDetails.passwd, dbSvrDetails.db) - ''' - Provides the Configuration Object to users through getConfigParser - The purpose of this object is to parse the config - and provide dictionary of the config so users can - use that configuration.Users can later call getConfig - on this object and it will return the default parsed - config dictionary from default configuration file, - they can overwrite it with providing their own - configuration file as well. - ''' - self.configObj = ConfigManager() - self.asyncJobMgr = None - self.id = None - self.defaultWorkerThreads = defaultWorkerThreads +class CSTestClient(object): + def __init__(self, mgmt_details, + dbsvr_details, + async_timeout=3600, + default_worker_threads=10, + logger=None, + test_data_filepath=None, + zone=None): + self.__mgmtDetails = mgmt_details + self.__dbSvrDetails = dbsvr_details + self.__csConnection = None + self.__dbConnection = None + self.__testClient = None + self.__asyncTimeOut = async_timeout + self.__logger = logger + self.__defaultWorkerThreads = default_worker_threads + self.__apiClient = None + self.__userApiClient = None + self.__asyncJobMgr = None + self.__id = None + self.__testDataFilePath = test_data_filepath + self.__parsedTestDataConfig = None + self.__zone = zone @property def identifier(self): - return self.id + return self.__id @identifier.setter def identifier(self, id): - self.id = id + self.__id = id - def createDbConnection(self, host="localhost", port=3306, user='cloud', - passwd='cloud', db='cloud'): - self.dbConnection = dbConnection.dbConnection(host, port, user, - passwd, db) + def getParsedTestDataConfig(self): + ''' + @Name : getParsedTestDataConfig + @Desc : Provides the TestData Config needed for + Tests are to Run + @Output : Returns the Parsed Test Data Dictionary + ''' + return self.__parsedTestDataConfig + + def getZoneForTests(self): + ''' + @Name : getZoneForTests + @Desc : Provides the Zone against which Tests are to run + If zone name provided to marvin plugin is none + it will get it from Test Data Config File + Even, if it is not available, return None + @Output : Returns the Zone Name + ''' + if self.__zone is None: + if self.__parsedTestDataConfig is not None: + self.__zone = self.__parsedTestDataConfig.get("zone") + return self.__zone + + def __setHypervisorToClient(self): + ''' + @Name : ___setHypervisorToClient + @Desc: Set the HyperVisor Details under API Client; + default to Xen + ''' + if self.__mgmtDetails.hypervisor: + self.__apiClient.hypervisor = self.__mgmtDetails.hypervisor + else: + self.__apiClient.hypervisor = XEN_SERVER + + def __createApiClient(self): + try: + ''' + Step1 : Create a CS Connection Object + ''' + self.__csConnection = CSConnection(self.__mgmtDetails, + self.__asyncTimeOut, + self.__logger) + + ''' + Step2 : Create API Client with earlier created connection object + ''' + self.__apiClient = CloudStackAPIClient(self.__csConnection) + + ''' + Step3: If API Key is not provided as part of Management Details, + then verify and register + ''' + if self.__mgmtDetails.apiKey is None: + list_user = listUsers.listUsersCmd() + list_user.account = "admin" + list_user_res = self.__apiClient.listUsers(list_user) + if list_user_res is None or\ + (validateList(list_user_res)[0] != PASS): + self.__logger.debug("__createApiClient: API " + "Client Creation Failed") + return FAILED + + user_id = list_user_res[0].id + api_key = list_user_res[0].apikey + security_key = list_user_res[0].secretkey + + if api_key is None: + ret = self.__getKeys(user_id) + if ret != FAILED: + self.__mgmtDetails.port = 8080 + self.__mgmtDetails.apiKey = ret[0] + self.__mgmtDetails.securityKey = ret[1] + else: + self.__logger.error("__createApiClient: API Client " + "Creation Failed while " + "Registering User") + return FAILED + ''' + Now Create the Connection objects and Api Client using + new details + ''' + self.__csConnection = CSConnection(self.__mgmtDetails, + self.__asyncTimeOut, + self.__logger) + self.__apiClient = CloudStackAPIClient(self.__csConnection) + ''' + Set the HyperVisor Details to Client default to Xen + ''' + self.__setHypervisorToClient() + return SUCCESS + except Exception, e: + self.__logger.exception(" Exception Occurred Under " + "__createApiClient: %s" % + GetDetailExceptionInfo(e)) + return FAILED + + def __createDbConnection(self): + ''' + @Name : ___createDbConnection + @Desc : Creates the CloudStack DB Connection + ''' + host = "localhost" if self.__dbSvrDetails.dbSvr is None \ + else self.__dbSvrDetails.dbSvr + port = 3306 if self.__dbSvrDetails.port is None \ + else self.__dbSvrDetails.port + user = "cloud" if self.__dbSvrDetails.user is None \ + else self.__dbSvrDetails.user + passwd = 'cloud' if self.__dbSvrDetails.passd is None \ + else self.__dbSvrDetails.passd + db = 'cloud' if self.__dbSvrDetails.db is None \ + else self.__dbSvrDetails.db + self.__dbConnection = DbConnection(host, port, user, passwd, db) + + def __getKeys(self, userid): + ''' + @Name : ___getKeys + @Desc : Retrieves the API and Secret Key for the provided Userid + ''' + try: + register_user = registerUserKeys.registerUserKeysCmd() + register_user.id = userid + register_user_res = \ + self.__apiClient.registerUserKeys(register_user) + + if register_user_res == FAILED: + return FAILED + return (register_user_res.apikey, register_user_res.secretkey) + except Exception, e: + self.__logger.exception("Exception Occurred Under __geKeys : " + "%s" % GetDetailExceptionInfo(e)) + return FAILED + + def createTestClient(self): + ''' + @Name : createTestClient + @Desc : Creates the Test Client. + The test Client is used by test suites + Here we create ParsedTestData Config. + Creates a DB Connection. + Creates an API Client + @Output : FAILED In case of an issue\Failure + SUCCESS in case of Success of this function + ''' + try: + ''' + 1. Create Config Object + Provides the Configuration Object to test suites through + getConfigParser. The purpose of this config object is to + parse the default config and provide dictionary of the + config so users can use that configuration. + Users can later call getConfig on this object and it will + return the default parsed config dictionary from default + configuration file. They can overwrite it with + providing their own configuration file as well. + ''' + self.__configObj = ConfigManager(self.__testDataFilePath) + if self.__configObj is not None: + self.__parsedTestDataConfig = self.__configObj.getConfig() + else: + self.__logger.error("createTestClient : Not able to create " + "ConfigManager Object") + return FAILED + ''' + 2. Create DB Connection + ''' + self.__createDbConnection() + ''' + 3. Creates API Client + ''' + return self.__createApiClient() + except Exception, e: + self.__logger.exception("Exception Occurred " + "Under createTestClient " + ": %s" % GetDetailExceptionInfo(e)) + return FAILED def isAdminContext(self): """ - A user is a regular user if he fails to listDomains; + @Name : isAdminContext + @Desc:A user is a regular user if he fails to listDomains; if he is a domain-admin, he can list only domains that are non-ROOT; if he is an admin, he can list the ROOT domain successfully """ try: listdom = listDomains.listDomainsCmd() listdom.name = 'ROOT' - listdomres = self.apiClient.listDomains(listdom) - rootdom = listdomres[0].name - if rootdom == 'ROOT': - return 1 # admin - else: - return 2 # domain-admin + listdomres = self.__apiClient.listDomains(listdom) + if listdomres != FAILED: + rootdom = listdomres[0].name + if rootdom == 'ROOT': + return ADMIN + else: + return DOMAIN_ADMIN + return USER except: - return 0 # user + return USER - def createUserApiClient(self, UserName, DomainName, acctType=0): - if not self.isAdminContext(): - return self.apiClient - - listDomain = listDomains.listDomainsCmd() - listDomain.listall = True - listDomain.name = DomainName + def __createUserApiClient(self, UserName, DomainName, acctType=0): + ''' + @Name : ___createUserApiClient + @Desc : Creates a User API Client with given + UserName\DomainName Parameters + ''' try: - domains = self.apiClient.listDomains(listDomain) - domId = domains[0].id - except: - cdomain = createDomain.createDomainCmd() - cdomain.name = DomainName - domain = self.apiClient.createDomain(cdomain) - domId = domain.id + if not self.isAdminContext(): + return self.__apiClient - cmd = listAccounts.listAccountsCmd() - cmd.name = UserName - cmd.domainid = domId - try: - accounts = self.apiClient.listAccounts(cmd) - acctId = accounts[0].id - except: - createAcctCmd = createAccount.createAccountCmd() - createAcctCmd.accounttype = acctType - createAcctCmd.domainid = domId - createAcctCmd.email = "test-" + random_gen()\ - + "@cloudstack.org" - createAcctCmd.firstname = UserName - createAcctCmd.lastname = UserName - createAcctCmd.password = 'password' - createAcctCmd.username = UserName - acct = self.apiClient.createAccount(createAcctCmd) - acctId = acct.id + listDomain = listDomains.listDomainsCmd() + listDomain.listall = True + listDomain.name = DomainName + try: + domains = self.__apiClient.listDomains(listDomain) + domId = domains[0].id + except: + cdomain = createDomain.createDomainCmd() + cdomain.name = DomainName + domain = self.__apiClient.createDomain(cdomain) + domId = domain.id - listuser = listUsers.listUsersCmd() - listuser.username = UserName + cmd = listAccounts.listAccountsCmd() + cmd.name = UserName + cmd.domainid = domId + try: + accounts = self.__apiClient.listAccounts(cmd) + acctId = accounts[0].id + except: + createAcctCmd = createAccount.createAccountCmd() + createAcctCmd.accounttype = acctType + createAcctCmd.domainid = domId + createAcctCmd.email = "test-" + random_gen()\ + + "@cloudstack.org" + createAcctCmd.firstname = UserName + createAcctCmd.lastname = UserName + createAcctCmd.password = 'password' + createAcctCmd.username = UserName + acct = self.__apiClient.createAccount(createAcctCmd) + acctId = acct.id - listuserRes = self.apiClient.listUsers(listuser) - userId = listuserRes[0].id - apiKey = listuserRes[0].apikey - securityKey = listuserRes[0].secretkey + listuser = listUsers.listUsersCmd() + listuser.username = UserName - if apiKey is None: - registerUser = registerUserKeys.registerUserKeysCmd() - registerUser.id = userId - registerUserRes = self.apiClient.registerUserKeys(registerUser) - apiKey = registerUserRes.apikey - securityKey = registerUserRes.secretkey + listuserRes = self.__apiClient.listUsers(listuser) + userId = listuserRes[0].id + apiKey = listuserRes[0].apikey + securityKey = listuserRes[0].secretkey - mgtDetails = self.mgmtDetails - mgtDetails.apiKey = apiKey - mgtDetails.securityKey = securityKey + if apiKey is None: + ret = self.__getKeys(userId) + if ret != FAILED: + mgtDetails = self.__mgmtDetails + mgtDetails.apiKey = ret[0] + mgtDetails.securityKey = ret[1] + else: + self.__logger.error("__createUserApiClient: " + "User API Client Creation." + " While Registering User Failed") + return FAILED - newUserConnection =\ - cloudstackConnection.cloudConnection(mgtDetails, - self.connection.asyncTimeout, - self.connection.logger) - self.userApiClient =\ - cloudstackAPIClient.CloudStackAPIClient(newUserConnection) - self.userApiClient.connection = newUserConnection - self.userApiClient.hypervisor = self.apiClient.hypervisor - return self.userApiClient + newUserConnection =\ + CSConnection(mgtDetails, + self.__csConnection.asyncTimeout, + self.__csConnection.logger) + self.__userApiClient = CloudStackAPIClient(newUserConnection) + self.__userApiClient.connection = newUserConnection + self.__userApiClient.hypervisor = self.__apiClient.hypervisor + return self.__userApiClient + except Exception, e: + self.__logger.exception("Exception Occurred " + "Under getUserApiClient : %s" % + GetDetailExceptionInfo(e)) + return FAILED def close(self): - if self.connection is not None: - self.connection.close() + if self.__csConnection is not None: + self.__csConnection.close() def getDbConnection(self): - return self.dbConnection + ''' + @Name : getDbConnection + @Desc : Retrieves the DB Connection Handle + ''' + return self.__dbConnection def getConfigParser(self): - return self.configObj + ''' + @Name : getConfigParser + @Desc : Provides the ConfigManager Interface to TestClients + ''' + return self.__configObj def getApiClient(self): - self.apiClient.id = self.identifier - return self.apiClient + if self.__apiClient is not None: + self.__apiClient.id = self.identifier + return self.__apiClient + return None def getUserApiClient(self, account, domain, type=0): """ - 0 - user - 1 - admin - 2 - domain admin + @Name : getUserApiClient + @Desc : Provides the User API Client to Users + 0 - user ; 1 - admin;2 - domain admin + @OutPut : FAILED In case of an issue + else User API Client """ - self.createUserApiClient(account, domain, type) - if hasattr(self, "userApiClient"): - return self.userApiClient - return None + return FAILED if (self.__createUserApiClient(account, + domain, + type) + == FAILED) \ + else self.__userApiClient def submitCmdsAndWait(self, cmds, workers=1): - '''FixME, httplib has issue if more than one thread submitted''' - if self.asyncJobMgr is None: - self.asyncJobMgr = asyncJobMgr.asyncJobMgr(self.apiClient, - self.dbConnection) - return self.asyncJobMgr.submitCmdsAndWait(cmds, workers) + ''' + @Desc : FixME, httplib has issue if more than one thread submitted + ''' + if self.__asyncJobMgr is None: + self.__asyncJobMgr = asyncJobMgr.asyncJobMgr(self.__apiClient, + self.__dbConnection) + return self.__asyncJobMgr.submitCmdsAndWait(cmds, workers) def submitJob(self, job, ntimes=1, nums_threads=10, interval=1): ''' - submit one job and execute the same job ntimes, with nums_threads - of threads + @Desc : submit one job and execute the same job + ntimes, with nums_threads of threads ''' - if self.asyncJobMgr is None: - self.asyncJobMgr = asyncJobMgr.asyncJobMgr(self.apiClient, - self.dbConnection) - self.asyncJobMgr.submitJobExecuteNtimes(job, ntimes, nums_threads, - interval) + if self.__asyncJobMgr is None: + self.__asyncJobMgr = asyncJobMgr.asyncJobMgr(self.__apiClient, + self.__dbConnection) + self.__asyncJobMgr.submitJobExecuteNtimes(job, ntimes, + nums_threads, + interval) def submitJobs(self, jobs, nums_threads=10, interval=1): - '''submit n jobs, execute them with nums_threads of threads''' - if self.asyncJobMgr is None: - self.asyncJobMgr = asyncJobMgr.asyncJobMgr(self.apiClient, - self.dbConnection) - self.asyncJobMgr.submitJobs(jobs, nums_threads, interval) + ''' + @Desc :submit n jobs, execute them with nums_threads + of threads + ''' + if self.__asyncJobMgr is None: + self.__asyncJobMgr = asyncJobMgr.asyncJobMgr(self.__apiClient, + self.__dbConnection) + self.__asyncJobMgr.submitJobs(jobs, nums_threads, interval) diff --git a/tools/marvin/marvin/codegenerator.py b/tools/marvin/marvin/codegenerator.py index e0f056f66a0..d4a81cfa967 100644 --- a/tools/marvin/marvin/codegenerator.py +++ b/tools/marvin/marvin/codegenerator.py @@ -42,7 +42,7 @@ class cloudStackCmd(object): self.response = [] -class codeGenerator(object): +class CodeGenerator(object): """ Apache CloudStack- marvin python classes can be generated from the json returned by API discovery or from the xml spec of commands generated by @@ -454,7 +454,7 @@ if __name__ == "__main__": print parser.print_help() exit(1) - cg = codeGenerator(folder) + cg = CodeGenerator(folder) if options.spec is not None: cg.generateCodeFromXML(apiSpecFile) elif options.endpoint is not None: diff --git a/tools/marvin/marvin/codes.py b/tools/marvin/marvin/codes.py index 74fb05da35c..2f606dc3dd9 100644 --- a/tools/marvin/marvin/codes.py +++ b/tools/marvin/marvin/codes.py @@ -47,3 +47,18 @@ YES = "yes" FAILED = "FAILED" UNKNOWN_ERROR = "Unknown Error" EXCEPTION = "EXCEPTION" +INVALID_RESPONSE = "Invalid Response" +''' +Async Job Related Codes +''' +JOB_INPROGRESS = 0 +JOB_SUCCEEDED = 1 +JOB_FAILED = 2 +JOB_CANCELLED = 3 +''' +User Related Codes +''' +ADMIN = 1 +DOMAIN_ADMIN = 2 +USER = 0 +XEN_SERVER = "XenServer" diff --git a/tools/marvin/marvin/config/__init__.py b/tools/marvin/marvin/config/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tools/marvin/marvin/config/test_data.cfg b/tools/marvin/marvin/config/test_data.cfg new file mode 100644 index 00000000000..ef0a6b31320 --- /dev/null +++ b/tools/marvin/marvin/config/test_data.cfg @@ -0,0 +1,160 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Use the common configs added such as account, network_offerings, domain, project, +# or add your own data if required separately for any test case +{ + "zone": "NA", + "domain": + { + "name": "domain" + } + , + "project": + { + "name": "Project", + "displaytext": "Test project" + }, + "account": { + "email": "test-account@test.com", + "firstname": "test", + "lastname": "test", + "username": "test-account", + "password": "password" + }, + "service_offering": { + "name": "Tiny Instance", + "displaytext": "Tiny Instance", + "cpunumber": 1, + "cpuspeed": 100, + "memory": 128 + }, + "isolated_network_offering": { + "name": "Network offering-DA services", + "displaytext": "Network offering-DA services", + "guestiptype": "Isolated", + "supportedservices": "Dhcp,Dns,SourceNat,PortForwarding,Vpn,Firewall,Lb,UserData,StaticNat", + "traffictype": "GUEST", + "availability": "Optional'", + "serviceProviderList": { + "Dhcp": "VirtualRouter", + "Dns": "VirtualRouter", + "SourceNat": "VirtualRouter", + "PortForwarding": "VirtualRouter", + "Vpn": "VirtualRouter", + "Firewall": "VirtualRouter", + "Lb": "VirtualRouter", + "UserData": "VirtualRouter", + "StaticNat": "VirtualRouter" + } + }, + "isolated_network": { + "name": "Isolated Network", + "displaytext": "Isolated Network" + }, + "virtual_machine": { + "displayname": "Test VM", + "username": "root", + "password": "password", + "ssh_port": 22, + "privateport": 22, + "publicport": 22, + "protocol": "TCP" + }, + "shared_network": { + "name": "MySharedNetwork - Test", + "displaytext": "MySharedNetwork", + "vlan" : "", + "gateway" :"", + "netmask" :"", + "startip" :"", + "endip" :"", + "acltype" : "Domain", + "scope":"all" + }, + "shared_network_offering_sg": { + "name": "MySharedOffering-sg", + "displaytext": "MySharedOffering-sg", + "guestiptype": "Shared", + "supportedservices": "Dhcp,Dns,UserData,SecurityGroup", + "specifyVlan" : "False", + "specifyIpRanges" : "False", + "traffictype": "GUEST", + "serviceProviderList" : { + "Dhcp": "VirtualRouter", + "Dns": "VirtualRouter", + "UserData": "VirtualRouter", + "SecurityGroup": "SecurityGroupProvider" + } + }, + "shared_network_sg": { + "name": "Shared-Network-SG-Test", + "displaytext": "Shared-Network_SG-Test", + "networkofferingid":"1", + "vlan" : "", + "gateway" :"", + "netmask" :"255.255.255.0", + "startip" :"", + "endip" :"", + "acltype" : "Domain", + "scope":"all" + }, + "vpc_offering": { + "name": "VPC off", + "displaytext": "VPC off", + "supportedservices": "Dhcp,Dns,SourceNat,PortForwarding,Vpn,Lb,UserData,StaticNat,NetworkACL" + }, + "vpc": { + "name": "TestVPC", + "displaytext": "TestVPC", + "cidr": "10.0.0.1/24" + }, + "shared_network_offering": { + "name": "MySharedOffering", + "displaytext": "MySharedOffering", + "guestiptype": "Shared", + "supportedservices": "Dhcp,Dns,UserData", + "specifyVlan" : "False", + "specifyIpRanges" : "False", + "traffictype": "GUEST", + "serviceProviderList" : { + "Dhcp": "VirtualRouter", + "Dns": "VirtualRouter", + "UserData": "VirtualRouter" + } + }, + "security_group" : { "name": "custom_Sec_Grp" }, + "ingress_rule": { + "protocol": "TCP", + "startport": "22", + "endport": "22", + "cidrlist": "0.0.0.0/0" + }, + "ostype": "CentOS 5.3 (64-bit)", + "sleep": 90, + "timeout": 10, + "advanced_sg": { + "zone": { + "name": "", + "dns1": "8.8.8.8", + "internaldns1": "192.168.100.1", + "networktype": "Advanced", + "securitygroupenabled": "true" + }, + "securitygroupenabled": "true" + } +} diff --git a/tools/marvin/marvin/configGenerator.py b/tools/marvin/marvin/configGenerator.py index 0d79e8ea6ad..245473a188e 100644 --- a/tools/marvin/marvin/configGenerator.py +++ b/tools/marvin/marvin/configGenerator.py @@ -50,9 +50,13 @@ class configuration(object): class logger(object): def __init__(self): - '''TestCase/TestClient''' - self.name = None - self.file = None + self.LogFolderPath = None + + +class apiLoadCfg(object): + def __init__(self): + self.ParsedApiDestFolder = None + self.ApiSpecFile = None class cloudstackConfiguration(object): @@ -296,9 +300,8 @@ class bigip(object): class ConfigManager(object): - ''' - @Name: configManager + @Name: ConfigManager @Desc: 1. It provides the basic configuration facilities to marvin. 2. User can just add configuration files for his tests, deployment etc, under one config folder before running their tests. @@ -328,17 +331,22 @@ class ConfigManager(object): 8. Users can use their own configuration file passed to "getConfig" API,once configObj is returned. ''' + def __init__(self, cfg_file=None): + if cfg_file is None: + self.__filePath = "config/test_data.cfg" + else: + self.__filePath = cfg_file + self.__parsedCfgDict = None + ''' + Set the Configuration + ''' + self.__setConfig() - def __init__(self): - # Joining path with current directory will avoid relative path issue - # It will take correct path irrespective of from where the test case is run - dirPath = os.path.dirname(__file__) - self.filePath = os.path.join(dirPath, 'config/config.cfg') - self.parsedDict = None - if self.__verifyFile(self.filePath) is not False: - self.parsedDict = self.__parseConfig(self.filePath) + def __setConfig(self): + if self.__verifyFile() is not False: + self.__parsedCfgDict = self.__parseConfig() - def __parseConfig(self, file): + def __parseConfig(self): ''' @Name : __parseConfig @Description: Parses the Input configuration Json file @@ -363,22 +371,20 @@ class ConfigManager(object): finally: return config_dict - def __verifyFile(self, file): + def __verifyFile(self): ''' @Name : __parseConfig @Description: Parses the Input configuration Json file and returns a dictionary from the file. - @Input : file NA + @Input : NA @Output : True or False based upon file input validity and availability ''' - if file is None or file == '': + if self.__filePath is None or self.__filePath == '': return False - if os.path.exists(file) is False: - return False - return True + return False if os.path.exists(self.__filePath) is False else True - def __getSectionData(self, return_dict, section=None): + def getSectionData(self, section=None): ''' @Name: getSectionData @Desc: Gets the Section data of a particular section @@ -387,35 +393,21 @@ class ConfigManager(object): section to be returned from this dict @Output:Section matching inside the parsed data ''' - if return_dict is not None: - inp = return_dict - elif self.parsedDict is None: + if self.__parsedCfgDict is None or section is None: + print "\nEither Parsed Dictionary is None or Section is None" return INVALID_INPUT - else: - inp = self.parsedDict - if section is not None: - return inp.get(section) - else: - return inp + return self.__parsedCfgDict.get(section) - def getConfig(self, file_path=None, section=None): + def getConfig(self): ''' - @Name: getConfig - @Desc : Parses and converts the given configuration file to dictionary - @Input : file_path: path where the configuration needs to be passed - section: specific section inside the file - @Output: INVALID_INPUT: This value is returned if the input - is invalid or not able to be parsed - Parsed configuration dictionary from json file + @Name : getConfig + @Desc : Returns the Parsed Dictionary of Config Provided + @Input : NA + @Output: ParsedDict if successful if cfg file provided is valid + None if cfg file is invalid or not able to be parsed ''' - ret = None - if file not in [None, '']: - if self.__verifyFile(file_path) is False: - return INVALID_INPUT - else: - ret = self.__parseConfig(file_path) - return self.__getSectionData(ret, section) + return self.__parsedCfgDict def getDeviceUrl(obj): diff --git a/tools/marvin/marvin/dbConnection.py b/tools/marvin/marvin/dbConnection.py index 99014abfa20..422fcfa8f21 100644 --- a/tools/marvin/marvin/dbConnection.py +++ b/tools/marvin/marvin/dbConnection.py @@ -25,7 +25,7 @@ import sys import os -class dbConnection(object): +class DbConnection(object): def __init__(self, host="localhost", port=3306, user='cloud', passwd='cloud', db='cloud'): self.host = host @@ -68,7 +68,7 @@ class dbConnection(object): return self.execute(sqls) if __name__ == "__main__": - db = dbConnection() + db = DbConnection() ''' try: diff --git a/tools/marvin/marvin/deployDataCenter.py b/tools/marvin/marvin/deployDataCenter.py index c4f6e1e1377..85a31d381f8 100644 --- a/tools/marvin/marvin/deployDataCenter.py +++ b/tools/marvin/marvin/deployDataCenter.py @@ -17,20 +17,28 @@ """Deploy datacenters according to a json configuration file""" import configGenerator -import cloudstackException -import cloudstackTestClient +from cloudstackException import ( + InvalidParameterException, + GetDetailExceptionInfo) + import logging from cloudstackAPI import * from os import path from time import sleep from optparse import OptionParser +from marvin.codes import (FAILED, SUCCESS) +from sys import exit -class deployDataCenters(object): - - def __init__(self, cfg, logger=None): +class DeployDataCenters(object): + ''' + @Desc : Deploys the Data Center with information provided + ''' + def __init__(self, test_client, cfg, logger=None): + self.testClient = test_client self.config = cfg self.tcRunLogger = logger + self.apiClient = None def addHosts(self, hosts, zoneId, podId, clusterId, hypervisor): if hosts is None: @@ -50,7 +58,9 @@ class deployDataCenters(object): hostcmd.username = host.username hostcmd.zoneid = zoneId hostcmd.hypervisor = hypervisor - self.apiClient.addHost(hostcmd) + if self.apiClient.addHost(hostcmd) == FAILED: + self.tcRunLogger.exception("=== Adding Host Failed===") + exit(1) def addVmWareDataCenter(self, vmwareDc): vdc = addVmwareDc.addVmwareDcCmd() @@ -59,12 +69,13 @@ class deployDataCenters(object): vdc.vcenter = vmwareDc.vcenter vdc.username = vmwareDc.username vdc.password = vmwareDc.password - self.apiClient.addVmwareDc(vdc) + if self.apiClient.addVmwareDc(vdc) == FAILED: + self.tcRunLogger.exception("=== Adding VmWare DC Failed===") + exit(1) def createClusters(self, clusters, zoneId, podId, vmwareDc=None): if clusters is None: return - if vmwareDc is not None: vmwareDc.zoneid = zoneId self.addVmWareDataCenter(vmwareDc) @@ -80,13 +91,22 @@ class deployDataCenters(object): clustercmd.username = cluster.username clustercmd.zoneid = zoneId clusterresponse = self.apiClient.addCluster(clustercmd) - clusterId = clusterresponse[0].id - + if clusterresponse != FAILED and clusterresponse[0].id is not None: + clusterId = clusterresponse[0].id + self.tcRunLogger.\ + debug("Cluster Name : %s Id : %s Created Successfully" + % (str(cluster.clustername), str(clusterId))) + else: + self.tcRunLogger.exception("====Cluster %s Creation Failed" + "=====" % str(cluster.clustername)) + exit(1) if cluster.hypervisor.lower() != "vmware": self.addHosts(cluster.hosts, zoneId, podId, clusterId, cluster.hypervisor) self.waitForHost(zoneId, clusterId) - self.createPrimaryStorages(cluster.primaryStorages, zoneId, podId, + self.createPrimaryStorages(cluster.primaryStorages, + zoneId, + podId, clusterId) def waitForHost(self, zoneId, clusterId): @@ -99,6 +119,9 @@ class deployDataCenters(object): cmd = listHosts.listHostsCmd() cmd.clusterid, cmd.zoneid = clusterId, zoneId hosts = self.apiClient.listHosts(cmd) + if hosts == FAILED: + self.tcRunLogger.exception("=== List Hosts Failed===") + exit(1) while retry != 0: for host in hosts: if host.state != 'Up': @@ -106,7 +129,11 @@ class deployDataCenters(object): sleep(timeout) retry = retry - 1 - def createPrimaryStorages(self, primaryStorages, zoneId, podId, clusterId): + def createPrimaryStorages(self, + primaryStorages, + zoneId, + podId, + clusterId): if primaryStorages is None: return for primary in primaryStorages: @@ -118,9 +145,15 @@ class deployDataCenters(object): primarycmd.url = primary.url primarycmd.zoneid = zoneId primarycmd.clusterid = clusterId - self.apiClient.createStoragePool(primarycmd) + if self.apiClient.createStoragePool(primarycmd) == FAILED: + self.tcRunLogger.\ + exception("=== Create Storage Pool Failed===") + exit(1) - def createPods(self, pods, zoneId, networkId=None): + def createPods(self, + pods, + zoneId, + networkId=None): if pods is None: return for pod in pods: @@ -132,7 +165,17 @@ class deployDataCenters(object): createpod.endip = pod.endip createpod.zoneid = zoneId createpodResponse = self.apiClient.createPod(createpod) - podId = createpodResponse.id + if createpodResponse != \ + FAILED and createpodResponse.id is not None: + podId = createpodResponse.id + self.tcRunLogger.debug("Pod Name : %s Id : %s " + "Created Successfully" % + (str(pod.name), str(podId))) + else: + self.tcRunLogger.\ + exception("====Pod: %s Creation " + "Failed=====" % str(pod.name)) + exit(1) if pod.guestIpRanges is not None and networkId is not None: self.createVlanIpRanges("Basic", pod.guestIpRanges, zoneId, @@ -164,7 +207,10 @@ class deployDataCenters(object): vlanipcmd.forvirtualnetwork = "false" else: vlanipcmd.forvirtualnetwork = "true" - self.apiClient.createVlanIpRange(vlanipcmd) + if self.apiClient.createVlanIpRange(vlanipcmd) == FAILED: + self.tcRunLogger.\ + exception("=== Create Vlan Ip Range Failed===") + exit(1) def createSecondaryStorages(self, secondaryStorages, zoneId): if secondaryStorages is None: @@ -184,7 +230,10 @@ class deployDataCenters(object): }) if secondarycmd.provider == "NFS": secondarycmd.zoneid = zoneId - self.apiClient.addImageStore(secondarycmd) + if self.apiClient.addImageStore(secondarycmd) == FAILED: + self.tcRunLogger.\ + exception("=== Add Image Store Failed===") + exit(1) def createCacheStorages(self, cacheStorages, zoneId): if cacheStorages is None: @@ -203,7 +252,11 @@ class deployDataCenters(object): 'key': key, 'value': value }) - self.apiClient.createSecondaryStagingStore(cachecmd) + if self.apiClient.createSecondaryStagingStore(cachecmd) == FAILED: + self.tcRunLogger.\ + exception("=== Create " + "SecondaryStagingStorage Failed===") + exit(1) def createNetworks(self, networks, zoneId): if networks is None: @@ -224,7 +277,17 @@ class deployDataCenters(object): networkcmd.netmask = iprange.netmask networkcmdresponse = self.apiClient.createNetwork(networkcmd) - networkId = networkcmdresponse.id + if networkcmdresponse != \ + FAILED and networkcmdresponse.id is not None: + networkId = networkcmdresponse.id + self.tcRunLogger.\ + debug("Network Name : %s Id : %s Created Successfully" + % (str(network.name), str(networkId))) + else: + self.tcRunLogger.\ + exception("====Network : %s " + "Creation Failed=====" % str(network.name)) + exit(1) return networkId def createPhysicalNetwork(self, net, zoneid): @@ -233,30 +296,48 @@ class deployDataCenters(object): phynet.name = net.name phynet.isolationmethods = net.isolationmethods phynetwrk = self.apiClient.createPhysicalNetwork(phynet) + if phynetwrk != FAILED and phynetwrk.id is not None: + self.tcRunLogger.debug("Physical Network Name : %s Id : %s " + "Created Successfully" % + (str(phynet.name), + str(phynetwrk.id))) + else: + self.tcRunLogger.exception("====Physical Network " + "Creation Failed=====") + exit(1) self.addTrafficTypes(phynetwrk.id, net.traffictypes) return phynetwrk - def updatePhysicalNetwork(self, networkid, state="Enabled", vlan=None): + def updatePhysicalNetwork(self, networkid, state="Enabled", + vlan=None): upnet = updatePhysicalNetwork.updatePhysicalNetworkCmd() upnet.id = networkid upnet.state = state if vlan: upnet.vlan = vlan - return self.apiClient.updatePhysicalNetwork(upnet) + ret = self.apiClient.updatePhysicalNetwork(upnet) + if ret == FAILED: + self.tcRunLogger.\ + exception("====Update Physical Network Failed=====") + exit(1) + else: + return ret def enableProvider(self, provider_id): upnetprov =\ updateNetworkServiceProvider.updateNetworkServiceProviderCmd() upnetprov.id = provider_id upnetprov.state = "Enabled" - self.apiClient.updateNetworkServiceProvider(upnetprov) + if self.apiClient.updateNetworkServiceProvider(upnetprov) == FAILED: + self.tcRunLogger.\ + exception("====Update Network Service Provider Failed=====") + exit(1) def configureProviders(self, phynetwrk, providers): """ We will enable the virtualrouter elements for all zones. Other providers like NetScalers, SRX, etc are explicitly added/configured """ - for provider in providers: pnetprov = listNetworkServiceProviders.\ listNetworkServiceProvidersCmd() @@ -264,6 +345,11 @@ class deployDataCenters(object): pnetprov.state = "Disabled" pnetprov.name = provider.name pnetprovres = self.apiClient.listNetworkServiceProviders(pnetprov) + if pnetprovres == FAILED: + self.tcRunLogger.\ + exception("====List Network " + "Service Providers Failed=====") + exit(1) if pnetprovres and len(pnetprovres) > 0: if provider.name == 'VirtualRouter'\ @@ -280,7 +366,13 @@ class deployDataCenters(object): configureVirtualRouterElementCmd() vrconfig.enabled = "true" vrconfig.id = vrprovid - self.apiClient.configureVirtualRouterElement(vrconfig) + if self.apiClient.\ + configureVirtualRouterElement(vrconfig) == \ + FAILED: + self.tcRunLogger.\ + exception("====ConfigureVirtualRouterElement " + "Failed=====") + exit(1) self.enableProvider(pnetprovres[0].id) elif provider.name == 'InternalLbVm': internallbprov = listInternalLoadBalancerElements.\ @@ -288,8 +380,13 @@ class deployDataCenters(object): internallbprov.nspid = pnetprovres[0].id internallbresponse = self.apiClient.\ listInternalLoadBalancerElements(internallbprov) + if internallbresponse == FAILED: + self.tcRunLogger.\ + exception("====List " + "InternalLoadBalancerElements " + "Failed=====") + exit(1) internallbid = internallbresponse[0].id - internallbconfig = \ configureInternalLoadBalancerElement.\ configureInternalLoadBalancerElementCmd() @@ -333,9 +430,10 @@ class deployDataCenters(object): dev.physicalnetworkid = phynetwrk.id self.apiClient.addF5LoadBalancer(dev) else: - raise cloudstackException.\ - InvalidParameterException("Device %s doesn't match\ - any know provider type" % device) + raise InvalidParameterException("Device %s " + "doesn't match " + "any know provider " + "type" % device) self.enableProvider(result.id) def addTrafficTypes(self, physical_network_id, traffictypes): @@ -354,19 +452,32 @@ class deployDataCenters(object): if traffictype.vmware is not None else None traffic_type.simulatorlabel = traffictype.simulator\ if traffictype.simulator is not None else None - return self.apiClient.addTrafficType(traffic_type) + ret = self.apiClient.addTrafficType(traffic_type) + if ret == FAILED: + self.tcRunLogger.\ + exception("==== Add TrafficType Failed=====") + else: + return ret def enableZone(self, zoneid, allocation_state="Enabled"): zoneCmd = updateZone.updateZoneCmd() zoneCmd.id = zoneid zoneCmd.allocationstate = allocation_state - return self.apiClient.updateZone(zoneCmd) + ret = self.apiClient.updateZone(zoneCmd) + if ret == FAILED: + self.tcRunLogger.exception("==== Update Zone Failed=====") + else: + return ret def updateZoneDetails(self, zoneid, details): zoneCmd = updateZone.updateZoneCmd() zoneCmd.id = zoneid zoneCmd.details = details - return self.apiClient.updateZone(zoneCmd) + ret = self.apiClient.updateZone(zoneCmd) + if ret == FAILED: + self.tcRunLogger.exception("==== Update Zone Failed=====") + else: + return ret def createZones(self, zones): for zone in zones: @@ -383,7 +494,16 @@ class deployDataCenters(object): createzone.guestcidraddress = zone.guestcidraddress zoneresponse = self.apiClient.createZone(createzone) - zoneId = zoneresponse.id + if zoneresponse != FAILED and zoneresponse.id is not None: + zoneId = zoneresponse.id + self.tcRunLogger.debug("Zone Name : %s Id : %s " + "Created Successfully" % + (str(zone.name), str(zoneId))) + else: + self.tcRunLogger.\ + exception("====ZoneCreation : %s Failed=====" % + str(zone.name)) + exit(1) for pnet in zone.physical_networks: phynetwrk = self.createPhysicalNetwork(pnet, zoneId) @@ -406,7 +526,11 @@ class deployDataCenters(object): listnetworkofferingresponse = \ self.apiClient.listNetworkOfferings(listnetworkoffering) - + if listnetworkofferingresponse == FAILED: + self.tcRunLogger.\ + exception("==== " + "ListNetworkOfferingResponse Failed=====") + exit(1) guestntwrk = configGenerator.network() guestntwrk.displaytext = "guestNetworkForBasicZone" guestntwrk.name = "guestNetworkForBasicZone" @@ -437,7 +561,11 @@ class deployDataCenters(object): listnetworkofferingresponse = \ self.apiClient.listNetworkOfferings(listnetworkoffering) - + if listnetworkofferingresponse == FAILED: + self.tcRunLogger.\ + exception("==== ListNetworkOfferingResponse " + "Failed=====") + exit(1) networkcmd = createNetwork.createNetworkCmd() networkcmd.displaytext = "Shared SG enabled network" networkcmd.name = "Shared SG enabled network" @@ -455,7 +583,16 @@ class deployDataCenters(object): networkcmd.vlan = iprange.vlan networkcmdresponse = self.apiClient.createNetwork(networkcmd) - networkId = networkcmdresponse.id + if networkcmdresponse != \ + FAILED and networkcmdresponse.id is not None: + networkId = networkcmdresponse.id + self.tcRunLogger.\ + debug("Network Id : %s " + "Created Successfully" % str(networkId)) + else: + self.tcRunLogger.\ + exception("====Network Creation Failed=====") + exit(1) self.createPods(zone.pods, zoneId, networkId) '''Note: Swift needs cache storage first''' @@ -479,64 +616,21 @@ class deployDataCenters(object): return True return False - def registerApiKey(self): - listuser = listUsers.listUsersCmd() - listuser.account = "admin" - listuserRes = self.testClient.getApiClient().listUsers(listuser) - userId = listuserRes[0].id - apiKey = listuserRes[0].apikey - securityKey = listuserRes[0].secretkey - if apiKey is None: - registerUser = registerUserKeys.registerUserKeysCmd() - registerUser.id = userId - registerUserRes = \ - self.testClient.getApiClient().registerUserKeys(registerUser) - - apiKey = registerUserRes.apikey - securityKey = registerUserRes.secretkey - - self.config.mgtSvr[0].port = 8080 - self.config.mgtSvr[0].apiKey = apiKey - self.config.mgtSvr[0].securityKey = securityKey - return apiKey, securityKey - - def loadCfg(self): - ''' Retrieving Management Server Connection Details ''' - mgtDetails = self.config.mgtSvr[0] - ''' Retrieving Database Connection Details''' - dbSvrDetails = self.config.dbSvr - - self.testClient = \ - cloudstackTestClient.\ - cloudstackTestClient(mgtDetails, - dbSvrDetails, - logger=self.tcRunLogger) - - if mgtDetails.apiKey is None: - mgtDetails.apiKey, mgtDetails.securityKey = self.registerApiKey() - mgtDetails.port = 8080 - self.testClient = \ - cloudstackTestClient.cloudstackTestClient( - mgtDetails, - dbSvrDetails, - logger=self.tcRunLogger) - + def setClient(self): self.apiClient = self.testClient.getApiClient() - """set hypervisor""" - if mgtDetails.hypervisor: - self.apiClient.hypervisor = mgtDetails.hypervisor - else: - self.apiClient.hypervisor = "XenServer" # Defaults to Xenserver def updateConfiguration(self, globalCfg): - if globalCfg is None: + if globalCfg is None or self.apiClient is None: return None for config in globalCfg: updateCfg = updateConfiguration.updateConfigurationCmd() updateCfg.name = config.name updateCfg.value = config.value - self.apiClient.updateConfiguration(updateCfg) + if self.apiClient.updateConfiguration(updateCfg) == FAILED: + self.tcRunLogger.\ + exception("===UpdateConfiguration Failed===") + exit(1) def copyAttributesToCommand(self, source, command): map(lambda attr: setattr(command, attr, getattr(source, attr, None)), @@ -548,28 +642,61 @@ class deployDataCenters(object): return command = addS3.addS3Cmd() self.copyAttributesToCommand(s3, command) - self.apiClient.addS3(command) + if self.apiClient.addS3(command) == FAILED: + self.tcRunLogger.exception("====AddS3 Failed===") + exit(1) def deploy(self): - self.loadCfg() - self.updateConfiguration(self.config.globalConfig) - self.createZones(self.config.zones) - self.configureS3(self.config.s3) + try: + self.setClient() + self.updateConfiguration(self.config.globalConfig) + self.createZones(self.config.zones) + self.configureS3(self.config.s3) + return SUCCESS + except Exception, e: + print "\nException Occurred Under deploy :%s" % \ + GetDetailExceptionInfo(e) + return FAILED if __name__ == "__main__": parser = OptionParser() parser.add_option("-i", "--input", action="store", - default="./datacenterCfg", dest="input", help="the path \ + default="./datacenterCfg", dest="input", + help="the path \ where the json config file generated, by default is \ ./datacenterCfg") (options, args) = parser.parse_args() - from marvin.marvinLog import MarvinLog - cfg = configGenerator.getSetupConfig(options.input) - log_obj = MarvinLog("CSLog") - tcRunLogger = log_obj.setLogHandler("/tmp/debug.log") - deploy = deployDataCenters(cfg, tcRunLogger) - deploy.deploy() + if options.input: + ''' + Imports the Modules Required + ''' + from marvin.marvinLog import MarvinLog + from marvin.cloudstackTestClient import CSTestClient + + cfg = configGenerator.getSetupConfig(options.input) + log_obj = MarvinLog("CSLog") + tcRunLogger = log_obj.setLogHandler("/tmp/debug.log") + if tcRunLogger is None: + print "\nLogger Creation Failed. " \ + "Please Check" + exit(1) + else: + print "\nAll Logs Are Available " \ + "Under /tmp/debug.log File" + obj_tc_client = CSTestClient(cfg.mgtSvr[0], cfg.dbSvr, + logger=tcRunLogger) + if obj_tc_client is not None and obj_tc_client.CreateTestClient() \ + != FAILED: + deploy = DeployDataCenters(obj_tc_client, cfg, tcRunLogger) + if deploy.deploy() == FAILED: + print "\nDeploy DC Failed" + exit(1) + else: + print "\nTestClient Creation Failed. Please Check" + exit(1) + else: + print "\n Please Specify a Valid Configuration File" """ create = createStoragePool.createStoragePoolCmd() diff --git a/tools/marvin/marvin/integration/lib/common.py b/tools/marvin/marvin/integration/lib/common.py index 550de1aca77..c16339b22ae 100644 --- a/tools/marvin/marvin/integration/lib/common.py +++ b/tools/marvin/marvin/integration/lib/common.py @@ -68,6 +68,10 @@ from marvin.integration.lib.utils import (get_process_status, from marvin.sshClient import SshClient import random +from utils import * +from base import * +from marvin.codes import PASS +from marvin.integration.lib.utils import validateList #Import System modules import time @@ -140,110 +144,155 @@ def add_netscaler(apiclient, zoneid, NSservice): return netscaler -def get_region(apiclient, services=None): - "Returns a default region" - +def get_region(apiclient, region_id=None, region_name=None): + ''' + @name : get_region + @Desc : Returns the Region Information for a given region id or region name + @Input : region_name: Name of the Region + region_id : Id of the region + @Output : 1. Region Information for the passed inputs else first Region + 2. FAILED In case the cmd failed + ''' + if region_id is None and region_name is None: + return FAILED cmd = listRegions.listRegionsCmd() - if services: - if "regionid" in services: - cmd.id = services["regionid"] + if region_name is not None: + cmd.name = region_name + if region_id is not None: + cmd.id = region_id + cmd_out = apiclient.listRegions(cmd) + return FAILED if validateList(cmd_out)[0] != PASS + return cmd_out - regions = apiclient.listRegions(cmd) - - if isinstance(regions, list): - assert len(regions) > 0 - return regions[0] - else: - raise Exception("Failed to find specified region.") - -def get_domain(apiclient, services=None): - "Returns a default domain" +def get_domain(apiclient, domain_id=None, domain_name=None): + ''' + @name : get_domain + @Desc : Returns the Domain Information for a given domain id or domain name + @Input : domain id : Id of the Domain + domain_name : Name of the Domain + @Output : 1. Domain Information for the passed inputs else first Domain + 2. FAILED In case the cmd failed + ''' cmd = listDomains.listDomainsCmd() - if services: - if "domainid" in services: - cmd.id = services["domainid"] - domains = apiclient.listDomains(cmd) - - if isinstance(domains, list): - assert len(domains) > 0 - return domains[0] - else: - raise Exception("Failed to find specified domain.") + if domain_name is not None: + cmd.name = domain_name + if domain_id is not None: + cmd.id = domain_id + cmd_out = apiclient.listRegions(cmd) + return FAILED if validateList(cmd_out)[0] != PASS + return cmd_out -def get_zone(apiclient, services=None): - "Returns a default zone" - +def get_zone(apiclient, zone_name=None, zone_id=None): + ''' + @name : get_zone + @Desc :Returns the Zone Information for a given zone id or Zone Name + @Input : zone_name: Name of the Zone + zone_id : Id of the zone + @Output : 1. Zone Information for the passed inputs else first zone + 2. FAILED In case the cmd failed + ''' cmd = listZones.listZonesCmd() - if services: - if "zoneid" in services: - cmd.id = services["zoneid"] + if zone_name is not None: + cmd.name = zone_name + if zone_id is not None: + cmd.id = zone_id - zones = apiclient.listZones(cmd) + cmd_out = apiclient.listZones(cmd) - if isinstance(zones, list): - assert len(zones) > 0, "There are no available zones in the deployment" - return zones[0] + return FAILED if (validateList(cmd_out)[0] != PASS) + ''' + Check if input zone name and zone id is None, + then return first element of List Zones command + ''' + if ( zone_name is None and zone_id is None ) + return cmd_out[0] else: - raise Exception("Failed to find specified zone.") + return cmd_out -def get_pod(apiclient, zoneid, services=None): - "Returns a default pod for specified zone" +def get_pod(apiclient, pod_id=None, pod_name=None, zone_id=None): + ''' + @name : get_pod + @Desc : Returns the Pod Information for a given zone id or Zone Name + @Input : pod_name : Name of the Pod + pod_id : Id of the Pod + zone_id: Id of the Zone + @Output : 1. Pod Information for the pod + 2. FAILED In case the cmd failed + ''' cmd = listPods.listPodsCmd() - cmd.zoneid = zoneid - if services: - if "podid" in services: - cmd.id = services["podid"] + if pod_name is not None: + cmd.name = pod_name + if pod_id is not None: + cmd.id = pod_id + if zone_id is not None: + cmd.zoneid = zone_id - pods = apiclient.listPods(cmd) + cmd_out = apiclient.listPods(cmd) - if isinstance(pods, list): - assert len(pods) > 0, "No pods found for zone %s"%zoneid - return pods[0] - else: - raise Exception("Exception: Failed to find specified pod.") + return FAILED if ( validateList(cmd_out)[0] != PASS ) + return cmd_out -def get_template(apiclient, zoneid, ostype, services=None, - templatefilter='featured', - templatetype='BUILTIN'): - "Returns a template" +def get_template(apiclient, template_id=None, template_name=None, account=None, template_type='BUILTIN' + domain_id=None, zone_id=None, project_id=None, + hypervisor=None, ostype_desc=None, template_filter="featured"): + ''' + @Name : get_template + @Desc : Retrieves the template Information based upon inputs provided + Template is retrieved based upon either of the inputs matched + condition + @Input : returns a template" + @Output : FAILED in case of any failure + template Information matching the inputs + ''' + ''' + Get OS TypeID First based upon ostype_desc + ''' cmd = listOsTypes.listOsTypesCmd() - cmd.description = ostype - ostypes = apiclient.listOsTypes(cmd) + cmd.description = ostype_desc + ostypes_out = apiclient.listOsTypes(cmd) - if isinstance(ostypes, list): - ostypeid = ostypes[0].id - else: - raise Exception( - "Failed to find OS type with description: %s" % ostype) + return FAILED if (validateList(ostypes_out)[0] != PASS ) - cmd = listTemplates.listTemplatesCmd() - cmd.templatefilter = templatefilter - cmd.zoneid = zoneid + ostype_id = ostypes_out[0].id - if services: - if "template" in services: - cmd.id = services["template"] + listcmd = listTemplates.listTemplatesCmd() + cmd.templatefilter = template_filter + if domain_id is not None: + cmd.domainid = domain_id + if zone_id is not None: + cmd.zoneid = zone_id + if template_id is not None: + cmd.id = template_id + if template_name is not None: + cmd.name = template_name + if hypervisor is not None: + cmd.hypervisor = hypervisor + if project_id is not None: + cmd.projectid = project_id + if account is not None: + cmd.account = account - list_templates = apiclient.listTemplates(cmd) - - if isinstance(list_templates, list): - assert len(list_templates) > 0, "received empty response on template of type %s"%ostype - for template in list_templates: - if template.ostypeid == ostypeid and template.isready and template.templatetype == templatetype: - return template - - raise Exception("Exception: Failed to find template of type %s with OSTypeID and which is in " - "ready state: %s" %(templatetype, ostypeid)) - return + ''' + Get the Templates pertaining + ''' + list_templatesout = apiclient.listTemplates(cmd) + return FAILED if validateList(list_templatesout)[0] != PASS + for template in list_templatesout: + if template.ostypeid == ostype_id and template.isready and template.templatetype == template_type: + return template + ''' + Return Failed if None of the templates matched + ''' + return FAILED def download_systemplates_sec_storage(server, services): """Download System templates on sec storage""" @@ -251,13 +300,13 @@ def download_systemplates_sec_storage(server, services): try: # Login to management server ssh = SshClient( - server["ipaddress"], - server["port"], - server["username"], - server["password"] - ) + server["ipaddress"], + server["port"], + server["username"], + server["password"] + ) except Exception: - raise Exception("SSH access failted for server with IP address: %s" % + raise Exception("SSH access failed for server with IP address: %s" % server["ipaddess"]) # Mount Secondary Storage on Management Server cmds = [ diff --git a/tools/marvin/marvin/jsonHelper.py b/tools/marvin/marvin/jsonHelper.py index ae40b8dabf0..f7db5ba7829 100644 --- a/tools/marvin/marvin/jsonHelper.py +++ b/tools/marvin/marvin/jsonHelper.py @@ -145,7 +145,7 @@ def getResultObj(returnObj, responsecls=None): errMsg = "errorCode: %s, errorText:%s" % (result.errorcode, result.errortext) respname = responseName.replace("response", "") - raise cloudstackException.cloudstackAPIException(respname, errMsg) + raise cloudstackException.CloudstackAPIException(respname, errMsg) if result.count is not None: for key in result.__dict__.iterkeys(): @@ -247,7 +247,7 @@ due to missing parameter jobid" }''' try: asynJob = getResultObj(result) - except cloudstackException.cloudstackAPIException, e: + except cloudstackException.CloudstackAPIException, e: print e result = '{ "queryasyncjobresultresponse" : {} }' diff --git a/tools/marvin/marvin/lib/__init__.py b/tools/marvin/marvin/lib/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tools/marvin/marvin/marvinInit.py b/tools/marvin/marvin/marvinInit.py index f722058a9fe..36d907ecbec 100644 --- a/tools/marvin/marvin/marvinInit.py +++ b/tools/marvin/marvin/marvinInit.py @@ -17,7 +17,7 @@ ''' @Desc: Initializes the marvin and does required prerequisites for starting it. -1. Parses the configuration file passed to marvin and creates a + 1. Parses the configuration file passed to marvin and creates a parsed config 2. Initializes the logging required for marvin.All logs are now made available under a single timestamped folder. @@ -28,8 +28,11 @@ for starting it. from marvin import configGenerator from marvin import cloudstackException from marvin.marvinLog import MarvinLog -from marvin.deployDataCenter import deployDataCenters +from marvin.deployDataCenter import DeployDataCenters +from marvin.cloudstackTestClient import CSTestClient +from marvin.cloudstackException import GetDetailExceptionInfo from marvin.codes import( + PASS, YES, NO, SUCCESS, @@ -41,17 +44,26 @@ import os import logging import string import random +from sys import exit +from marvin.codegenerator import CodeGenerator class MarvinInit: - def __init__(self, config_file, load_flag, log_folder_path=None): + def __init__(self, config_file, load_api_flag=None, + deploy_dc_flag=None, + test_module_name=None, + zone=None): self.__configFile = config_file - self.__loadFlag = load_flag + self.__deployFlag = deploy_dc_flag + self.__loadApiFlag = load_api_flag self.__parsedConfig = None - self.__logFolderPath = log_folder_path + self.__logFolderPath = None self.__tcRunLogger = None self.__testClient = None - self.__tcRunDebugFile = None + self.__tcResultFile = None + self.__testModuleName = test_module_name + self.__testDataFilePath = None + self.__zoneForTests = None def __parseConfig(self): ''' @@ -59,12 +71,15 @@ class MarvinInit: the parsed configuration ''' try: + if self.__configFile is None: + return FAILED self.__parsedConfig = configGenerator.\ getSetupConfig(self.__configFile) return SUCCESS except Exception, e: - print "\n Exception Occurred Under __parseConfig : %s" % str(e) - return None + print "\nException Occurred Under __parseConfig : " \ + "%s" % GetDetailExceptionInfo(e) + return FAILED def getParsedConfig(self): return self.__parsedConfig @@ -79,10 +94,14 @@ class MarvinInit: return self.__tcRunLogger def getDebugFile(self): - return self.__tcRunDebugFile + if self.__logFolderPath is None: + self.__tcResultFile = open(self.__logFolderPath + + "/results.txt", "w") + return self.__tcResultFile def init(self): ''' + @Name : init @Desc :Initializes the marvin by 1. Parsing the configuration and creating a parsed config structure @@ -91,14 +110,18 @@ class MarvinInit: 3. Creates the DataCenter based upon configuration provided ''' try: - if ((self.__parseConfig() is not None) and - (self.__initLogging() is not None) and - (self.__deployDC() is not None)): + if ((self.__parseConfig() != FAILED) and + (self.__setTestDataPath() != FAILED) and + (self.__initLogging() != FAILED) and + (self.__createTestClient() != FAILED) and + (self.__deployDC() != FAILED) and + (self.__loadNewApiFromXml() != FAILED)): return SUCCESS else: return FAILED except Exception, e: - print "\n Exception Occurred Under init %s" % str(e) + print "\n Exception Occurred Under init " \ + "%s" % GetDetailExceptionInfo(e) return FAILED def __initLogging(self): @@ -113,57 +136,96 @@ class MarvinInit: for a given test run are available under a given timestamped folder ''' - temp_path = "".join(str(time.time()).split(".")) - if self.__logFolderPath is None: - log_config = self.__parsedConfig.logger - if log_config is not None: - if log_config.LogFolderPath is not None: - self.logFolderPath = log_config.LogFolderPath + '/' \ - + temp_path - else: - self.logFolderPath = temp_path - else: - self.logFolderPath = temp_path - else: - self.logFolderPath = self.__logFolderPath + '/' + temp_path - if os.path.exists(self.logFolderPath): - self.logFolderPath += ''.join(random.choice( - string.ascii_uppercase + - string.digits for x in range(3))) - os.makedirs(self.logFolderPath) - ''' - Log File Paths - ''' - tc_failed_exceptionlog = self.logFolderPath + "/failed_" \ - "plus_" \ - "exceptions.txt" - tc_run_log = self.logFolderPath + "/runinfo.txt" - self.__tcRunDebugFile = open(self.logFolderPath + - "/results.txt", "w") - log_obj = MarvinLog("CSLog") - self.__tcRunLogger = log_obj.setLogHandler(tc_run_log) - log_obj.setLogHandler(tc_failed_exceptionlog, - log_level=logging.FATAL) + if log_obj is None: + return FAILED + else: + ret = log_obj.\ + getLogs(self.__testModuleName, + self.__parsedConfig.logger) + if ret != FAILED: + self.__logFolderPath = log_obj.getLogFolderPath() + self.__tcRunLogger = log_obj.getLogger() return SUCCESS except Exception, e: - print "\n Exception Occurred Under __initLogging :%s" % str(e) - return None + print "\n Exception Occurred Under __initLogging " \ + ":%s" % GetDetailExceptionInfo(e) + return FAILED + + def __createTestClient(self): + ''' + @Name : __createTestClient + @Desc : Creates the TestClient during init + based upon the parameters provided + ''' + try: + mgt_details = self.__parsedConfig.mgtSvr[0] + dbsvr_details = self.__parsedConfig.dbSvr + self.__testClient = CSTestClient(mgt_details, dbsvr_details, + logger=self.__tcRunLogger, + test_data_filepath= + self.__testDataFilePath, + zone=self.__zoneForTests) + if self.__testClient is not None: + return self.__testClient.createTestClient() + else: + return FAILED + except Exception, e: + print "\n Exception Occurred Under __createTestClient : %s" % \ + GetDetailExceptionInfo(e) + return FAILED + + def __loadNewApiFromXml(self): + try: + if self.__loadApiFlag: + apiLoadCfg = self.__parsedConfig.apiLoadCfg + api_dst_dir = apiLoadCfg.ParsedApiDestFolder + "/cloudstackAPI" + api_spec_file = apiLoadCfg.ApiSpecFile + + if not os.path.exists(api_dst_dir): + try: + os.mkdir(api_dst_dir) + except Exception, e: + print "Failed to create folder %s, " \ + "due to %s" % (api_dst_dir, + GetDetailExceptionInfo(e)) + exit(1) + mgt_details = self.__parsedConfig.mgtSvr[0] + cg = CodeGenerator(api_dst_dir) + if os.path.exists(api_spec_file): + cg.generateCodeFromXML(api_spec_file) + elif mgt_details is not None: + endpoint_url = 'http://%s:8096/client/api?' \ + 'command=listApis&response=json' \ + % mgt_details.mgtSvrIp + cg.generateCodeFromJSON(endpoint_url) + return SUCCESS + except Exception, e: + print "\n Exception Occurred Under __loadNewApiFromXml : %s" \ + % GetDetailExceptionInfo(e) + return FAILED + + def __setTestDataPath(self): + try: + if ((self.__parsedConfig.TestData is not None) and + (self.__parsedConfig.TestData.Path is not None)): + self.__testDataFilePath = self.__parsedConfig.TestData.Path + return SUCCESS + except Exception, e: + print "\nException Occurred Under __setTestDataPath : %s" % \ + GetDetailExceptionInfo(e) + return FAILED def __deployDC(self): try: ''' Deploy the DataCenter and retrieves test client. ''' - deploy_obj = deployDataCenters(self.__parsedConfig, + deploy_obj = DeployDataCenters(self.__testClient, + self.__parsedConfig, self.__tcRunLogger) - if self.__loadFlag: - deploy_obj.loadCfg() - else: - deploy_obj.deploy() - - self.__testClient = deploy_obj.testClient - return SUCCESS + return deploy_obj.deploy() if self.__deployFlag else FAILED except Exception, e: - print "\n Exception Occurred Under __deployDC : %s" % str(e) - return None + print "\n Exception Occurred Under __deployDC : %s" % \ + GetDetailExceptionInfo(e) + return FAILED diff --git a/tools/marvin/marvin/marvinLog.py b/tools/marvin/marvin/marvinLog.py index 76de185421b..53fbc6280db 100644 --- a/tools/marvin/marvin/marvinLog.py +++ b/tools/marvin/marvin/marvinLog.py @@ -20,17 +20,20 @@ import logging import sys import time -from marvin.codes import (NO, - YES +import os +from marvin.codes import (SUCCESS, + FAILED ) +from marvin.cloudstackException import GetDetailExceptionInfo class MarvinLog: ''' + @Name : MarvinLog @Desc : provides interface for logging to marvin @Input : logger_name : name for logger ''' - logFormat = logging.Formatter("%(asctime)s - %(levelname)s - %(name)s - %(message)s") + logFormat = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s") _instance = None def __new__(cls, logger_name): @@ -39,13 +42,28 @@ class MarvinLog: return cls._instance def __init__(self, logger_name): - self.loggerName = logger_name - self.logger = None + ''' + @Name: __init__ + @Input: logger_name for logger + ''' + self.__loggerName = logger_name + ''' + Logger for Logging Info + ''' + self.__logger = None + ''' + Log Folder Directory + ''' + self.__logFolderDir = None self.__setLogger() def __setLogger(self): - self.logger = logging.getLogger(self.loggerName) - self.logger.setLevel(logging.DEBUG) + ''' + @Name : __setLogger + @Desc : Sets the Logger and Level + ''' + self.__logger = logging.getLogger(self.__loggerName) + self.__logger.setLevel(logging.DEBUG) def setLogHandler(self, log_file_path, log_format=None, log_level=logging.DEBUG): @@ -54,6 +72,7 @@ class MarvinLog: @Input: log_file_path: Log File Path as where to store the logs log_format : Format of log messages to be dumped log_level : Determines the level of logging for this logger + @Output: SUCCESS if no issues else FAILED ''' try: if log_file_path is not None: @@ -66,8 +85,89 @@ class MarvinLog: else: stream.setFormatter(self.__class__.logFormat) stream.setLevel(log_level) - self.logger.addHandler(stream) + self.__logger.addHandler(stream) + return SUCCESS except Exception, e: - print "\n Exception Occurred Under setLogHandler %s" % str(e) - finally: - return self.logger + print "\nException Occurred Under " \ + "setLogHandler %s" % GetDetailExceptionInfo(e) + return FAILED + + def __cleanPreviousLogs(self, logfolder_to_remove): + ''' + @Name : __cleanPreviousLogs + @Desc : Removes the Previous Logs + @Return: N\A + @Input: logfolder_to_remove: Path of Log to remove + ''' + os.rmdir(logfolder_to_remove) + + def getLogger(self): + ''' + @Name:getLogger + @Desc : Returns the Logger + ''' + return self.__logger + + def getLogFolderPath(self): + ''' + @Name : getLogFolderPath + @Desc : Returns the final log directory path for marvin run + ''' + return self.__logFolderDir + + def createLogs(self, test_module_name=None, log_cfg=None): + ''' + @Name : createLogs + @Desc : Gets the Logger with file paths initialized and created + @Inputs :test_module_name: Test Module Name to use for logs while + creating log folder path + log_cfg: Log Configuration provided inside of + Configuration + @Output : SUCCESS\FAILED + ''' + try: + if log_cfg is None: + print "\nInvalid Log Folder Configuration." \ + "Please Check Config File" + return FAILED + if test_module_name is None: + temp_path = time.strftime("%b_%d_%Y_%H_%M_%S", + time.localtime()) + else: + temp_path = test_module_name + + if (('LogFolderPath' in log_cfg.__dict__.keys()) and + (log_cfg.__dict__.get('LogFolderPath') is not None)): + self.__cleanPreviousLogs(log_cfg. + __dict__. + get('LogFolderPath') + "MarvinLogs/") + temp_dir = log_cfg.__dict__.get('LogFolderPath') + "MarvinLogs" + else: + temp_dir = "MarvinLogs" + + self.__logFolderDir = temp_dir + temp_path + print "\n*********Log Folder Path: %s. " \ + "All logs will be available here **************" \ + % str(self.__logFolderDir) + + os.makedirs(self.__logFolderDir) + + ''' + Log File Paths + 1. FailedExceptionLog + 2. RunLog contains the complete Run Information for Test Run + 3. ResultFile contains the TC result information for Test Run + ''' + tc_failed_exception_log = \ + self.__logFolderDir + "/failed_plus_exceptions.txt" + tc_run_log = self.__logFolderDir + "/runinfo.txt" + if self.setLogHandler(tc_run_log, + log_level=logging.DEBUG) != FAILED: + self.setLogHandler(tc_failed_exception_log, + log_level=logging.FATAL) + return SUCCESS + return FAILED + except Exception, e: + print "\n Exception Occurred Under createLogs :%s" % \ + GetDetailExceptionInfo(e) + return FAILED diff --git a/tools/marvin/marvin/marvinPlugin.py b/tools/marvin/marvin/marvinPlugin.py index df7d7a3123a..f36bca87ca0 100644 --- a/tools/marvin/marvin/marvinPlugin.py +++ b/tools/marvin/marvin/marvinPlugin.py @@ -24,10 +24,8 @@ from marvin.marvinInit import MarvinInit from nose.plugins.base import Plugin from marvin.codes import (SUCCESS, FAILED, - EXCEPTION, - UNKNOWN_ERROR - ) -import traceback + EXCEPTION) +from marvin.cloudstackException import GetDetailExceptionInfo import time import os @@ -43,8 +41,22 @@ class MarvinPlugin(Plugin): self.identifier = None self.testClient = None self.parsedConfig = None - self.configFile = None - self.loadFlag = None + ''' + Contains Config File + ''' + self.__configFile = None + ''' + Signifies the flag whether to load new API Information + ''' + self.__loadNewApiFlag = None + ''' + Signifies the Zone against which all tests will be Run + ''' + self.__zoneForTests = None + ''' + Signifies the flag whether to deploy the New DC or Not + ''' + self.__deployDcFlag self.conf = None self.debugStream = sys.stdout self.testRunner = None @@ -66,14 +78,12 @@ class MarvinPlugin(Plugin): return else: self.enabled = True - self.configFile = options.config_file - self.loadFlag = options.load - self.logFolderPath = options.log_folder_path + + self.__configFile = options.config_file + self.__loadNewApiFlag = options.loadNewApiFlag + self.__deployDcFlag = options.deployDc + self.__zoneForTests = options.zone self.conf = conf - ''' - Initializes the marvin with required settings - ''' - self.startMarvin() def options(self, parser, env): """ @@ -83,19 +93,31 @@ class MarvinPlugin(Plugin): default=env.get('MARVIN_CONFIG', './datacenter.cfg'), dest="config_file", - help="Marvin's configuration file where the " + - "datacenter information is specified" + - " [MARVIN_CONFIG]") - parser.add_option("--load", action="store_true", + help="Marvin's configuration file is required." + "The config file containing the datacenter and " + "other management server " + "information is specified") + parser.add_option("--deploy-dc", action="store_true", default=False, - dest="load", - help="Only load the deployment configuration given") - parser.add_option("--log-folder-path", - action="store", + dest="deployDc", + help="Deploys the DC with Given Configuration." + "Requires only when DC needs to be deployed") + parser.add_option("--zone", action="zone_tests", default=None, - dest="log_folder_path", - help="Path to the folder " - "where log files will be stored") + dest="zone", + help="Runs all tests against this specified zone") + parser.add_option("--load-new-apis", action="store_true", + default=False, + dest="loadNewApiFlag", + help="Loads the New Apis with Given Api Xml File." + "Creates the new Api's from commands.xml File") + ''' + Check if the configuration file is not valid,print and exit + ''' + (options, args) = parser.parse_args() + if options.config_file is None: + parser.print_usage() + sys.exit(1) Plugin.options(self, parser, env) def wantClass(self, cls): @@ -105,16 +127,44 @@ class MarvinPlugin(Plugin): return True return None + def prepareTest(self, test): + ''' + @Desc : Initializes the marvin with required settings + ''' + test_module_name = test.__str__() + if self.startMarvin(test_module_name) == FAILED: + print "Starting Marvin FAILED. Please Check Config and " \ + "Arguments Supplied" + + def __checkImport(self, filename): + ''' + @Desc : Verifies to Import the test Module before running and check + whether if it is importable. + This will check for test modules which has some issues to be + getting imported. + Returns False or True based upon the result. + ''' + try: + __import__(filename) + return True + except ImportError, e: + self.tcRunLogger.exception("Module : %s Import " + "Failed Reason :%s" + % (filename, GetDetailExceptionInfo(e))) + return False + def wantFile(self, filename): ''' - Only python files will be used as test modules + @Desc : Only python files will be used as test modules ''' + if filename is None or filename == '': + return False parts = filename.split(os.path.sep) base, ext = os.path.splitext(parts[-1]) - if ext == '.py': - return True - else: + if ext != '.py': return False + else: + return self.__checkImport(filename) def loadTestsFromTestCase(self, cls): if cls.__name__ != 'cloudstackTestCase': @@ -134,24 +184,15 @@ class MarvinPlugin(Plugin): Currently used to record start time for tests Dump Start Msg of TestCase to Log """ - self.tcRunLogger.debug("::::::::::::STARTED : TC: " + + self.tcRunLogger.debug("\n\n::::::::::::STARTED : TC: " + str(self.testName) + " :::::::::::") self.startTime = time.time() - def getErrorInfo(self, err): - ''' - Extracts and returns the sanitized error message - ''' - if err is not None: - return str(traceback.format_exc()) - else: - return UNKNOWN_ERROR - def handleError(self, test, err): ''' Adds Exception throwing test cases and information to log. ''' - err_msg = self.getErrorInfo(err) + err_msg = GetDetailExceptionInfo(err) self.tcRunLogger.fatal("%s: %s: %s" % (EXCEPTION, self.testName, err_msg)) self.testResult = EXCEPTION @@ -160,12 +201,12 @@ class MarvinPlugin(Plugin): ''' Adds Failing test cases and information to log. ''' - err_msg = self.getErrorInfo(err) + err_msg = GetDetailExceptionInfo(err) self.tcRunLogger.fatal("%s: %s: %s" % (FAILED, self.testName, err_msg)) self.testResult = FAILED - def startMarvin(self): + def startMarvin(self, test_module_name): ''' Initializes the Marvin creates the test Client @@ -174,9 +215,11 @@ class MarvinPlugin(Plugin): Creates a debugstream for tc debug log ''' try: - obj_marvininit = MarvinInit(self.configFile, - self.loadFlag, - self.logFolderPath) + obj_marvininit = MarvinInit(self.__configFile, + self.__loadNewApiFlag, + self.__deployDcFlag, + test_module_name, + self.__zoneForoTests) if obj_marvininit.init() == SUCCESS: self.testClient = obj_marvininit.getTestClient() self.tcRunLogger = obj_marvininit.getLogger() @@ -191,7 +234,8 @@ class MarvinPlugin(Plugin): else: return FAILED except Exception, e: - print "Exception Occurred under startMarvin: %s" % str(e) + print "Exception Occurred under startMarvin: %s" % \ + GetDetailExceptionInfo(e) return FAILED def stopTest(self, test): @@ -221,5 +265,6 @@ class MarvinPlugin(Plugin): setattr(test, "clstestclient", self.testClient) if hasattr(test, "user"): # when the class-level attr applied. all test runs as 'user' - self.testClient.createUserApiClient(test.UserName, test.DomainName, - test.AcctType) + self.testClient.getUserApiClient(test.UserName, + test.DomainName, + test.AcctType) diff --git a/tools/marvin/marvin/src/__init__.py b/tools/marvin/marvin/src/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tools/marvin/marvin/tcExecuteEngine.py b/tools/marvin/marvin/tcExecuteEngine.py index f959e7e935f..c4373372e62 100644 --- a/tools/marvin/marvin/tcExecuteEngine.py +++ b/tools/marvin/marvin/tcExecuteEngine.py @@ -61,9 +61,10 @@ class TestCaseExecuteEngine(object): setattr(test.__class__, "clstestclient", self.testclient) if hasattr(test, "user"): # attribute when test is entirely executed as user - self.testclient.createUserApiClient(test.UserName, - test.DomainName, - test.AcctType) + self.testclient.\ + getUserApiClient(test.UserName, + test.DomainName, + test.AcctType) def run(self): if self.suite: