From 7ed81a6990956ef577c8e193fadb195546ed3181 Mon Sep 17 00:00:00 2001 From: Wilder Rodrigues Date: Mon, 5 Oct 2015 13:26:55 +0200 Subject: [PATCH] CLOUDSTACK-8915 - Making sure cleanup resources passes - Due to an issue with VPC routers (CLOUDSTACK-8935) we are not able to destroy networks before destroying the routers - Added a forcestop/destroy routers inside the tearDown to make sure it passes. The issue will be addressed in a separate PR - Make sure the routers list is cleaned after destroy_routers() is called - Populate routers list after the router is recreated --- .../test_routers_iptables_default_policy.py | 73 ++++++++++++------ .../component/test_routers_network_ops.py | 19 ++--- .../component/test_vpc_redundant.py | 63 ++++++++++++---- .../component/test_vpc_router_nics.py | 75 +++++++++++-------- 4 files changed, 154 insertions(+), 76 deletions(-) diff --git a/test/integration/component/test_routers_iptables_default_policy.py b/test/integration/component/test_routers_iptables_default_policy.py index b72e45faa61..a1bec4cac36 100644 --- a/test/integration/component/test_routers_iptables_default_policy.py +++ b/test/integration/component/test_routers_iptables_default_policy.py @@ -230,21 +230,19 @@ class TestVPCIpTablesPolicies(cloudstackTestCase): admin=True, domainid=cls.domain.id) - cls._cleanup = [cls.account] - cls.service_offering = ServiceOffering.create( cls.apiclient, cls.services["service_offering"]) - cls._cleanup.append(cls.service_offering) cls.logger = logging.getLogger('TestVPCIpTablesPolicies') cls.stream_handler = logging.StreamHandler() cls.logger.setLevel(logging.DEBUG) cls.logger.addHandler(cls.stream_handler) - cls.entity_manager = EntityManager(cls.apiclient, cls.services, cls.service_offering, cls.account, cls.zone, cls._cleanup, cls.logger) + cls.entity_manager = EntityManager(cls.apiclient, cls.services, cls.service_offering, cls.account, cls.zone, cls.logger) + cls._cleanup = [cls.service_offering, cls.account] return @classmethod @@ -274,6 +272,16 @@ class TestVPCIpTablesPolicies(cloudstackTestCase): account=self.account.name, domainid=self.account.domainid) + self.cleanup = [self.vpc, self.vpc_off] + self.entity_manager.set_cleanup(self.cleanup) + return + + def tearDown(self): + try: + self.entity_manager.destroy_routers() + cleanup_resources(self.apiclient, self.cleanup) + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) return @attr(tags=["advanced", "intervlan"], required_hardware="true") @@ -361,21 +369,18 @@ class TestRouterIpTablesPolicies(cloudstackTestCase): admin=True, domainid=cls.domain.id) - cls._cleanup = [cls.account] - cls.service_offering = ServiceOffering.create( cls.apiclient, cls.services["service_offering"]) - cls._cleanup.append(cls.service_offering) - cls.logger = logging.getLogger('TestRouterIpTablesPolicies') cls.stream_handler = logging.StreamHandler() cls.logger.setLevel(logging.DEBUG) cls.logger.addHandler(cls.stream_handler) - cls.entity_manager = EntityManager(cls.apiclient, cls.services, cls.service_offering, cls.account, cls.zone, cls._cleanup, cls.logger) + cls.entity_manager = EntityManager(cls.apiclient, cls.services, cls.service_offering, cls.account, cls.zone, cls.logger) + cls._cleanup = [cls.service_offering, cls.account] return @classmethod @@ -386,6 +391,18 @@ class TestRouterIpTablesPolicies(cloudstackTestCase): raise Exception("Warning: Exception during cleanup : %s" % e) return + def setUp(self): + self.cleanup = [] + self.entity_manager.set_cleanup(self.cleanup) + return + + def tearDown(self): + try: + cleanup_resources(self.apiclient, self.cleanup) + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) + return + @attr(tags=["advanced", "intervlan"], required_hardware="true") def test_02_routervm_iptables_policies(self): """ Test iptables default INPUT/FORWARD policy on RouterVM """ @@ -442,18 +459,21 @@ class TestRouterIpTablesPolicies(cloudstackTestCase): class EntityManager(object): - def __init__(self, apiclient, services, service_offering, account, zone, cleanup, logger): + def __init__(self, apiclient, services, service_offering, account, zone, logger): self.apiclient = apiclient self.services = services self.service_offering = service_offering self.account = account self.zone = zone - self.cleanup = cleanup self.logger = logger + self.cleanup = [] self.networks = [] self.routers = [] self.ips = [] + + def set_cleanup(self, cleanup): + self.cleanup = cleanup def add_nat_rules(self, vpc_id): for o in self.networks: @@ -514,7 +534,6 @@ class EntityManager(object): conservemode=False) nw_off.update(self.apiclient, state='Enabled') - self.cleanup.append(nw_off) self.logger.debug('Created and Enabled NetworkOffering') self.services["network"]["name"] = "NETWORK-" + str(gateway) @@ -528,13 +547,18 @@ class EntityManager(object): zoneid=self.zone.id, gateway=gateway, vpcid=vpc_id) + self.logger.debug("Created network with ID: %s" % obj_network.id) except Exception, e: raise Exception('Unable to create a Network with offering=%s because of %s ' % (net_offerring, e)) o = networkO(obj_network) - o.add_vm(self.deployvm_in_network(obj_network)) + vm1 = self.deployvm_in_network(obj_network) + self.cleanup.insert(1, obj_network) + self.cleanup.insert(2, nw_off) + + o.add_vm(vm1) self.networks.append(o) return o @@ -548,7 +572,9 @@ class EntityManager(object): domainid=self.account.domainid, serviceofferingid=self.service_offering.id, networkids=[str(network.id)]) + self.logger.debug('Created VM=%s in network=%s' % (vm.id, network.name)) + self.cleanup.insert(0, vm) return vm except: raise Exception('Unable to create VM in a Network=%s' % network.name) @@ -562,6 +588,8 @@ class EntityManager(object): accountid=self.account.name, domainid=self.account.domainid, serviceofferingid=self.service_offering.id) + + self.cleanup.insert(0, vm) self.logger.debug('Created VM=%s' % vm.id) return vm except: @@ -590,22 +618,23 @@ class EntityManager(object): return self.routers - def stop_router(self): + def stop_router(self, router): self.logger.debug('Stopping router') - for router in self.routers: - cmd = stopRouter.stopRouterCmd() - cmd.id = router.id - self.apiclient.stopRouter(cmd) + cmd = stopRouter.stopRouterCmd() + cmd.id = router.id + self.apiclient.stopRouter(cmd) - def destroy_router(self): - self.logger.debug('Destroying router') + def destroy_routers(self): + self.logger.debug('Destroying routers') for router in self.routers: + self.stop_router(router) cmd = destroyRouter.destroyRouterCmd() cmd.id = router.id self.apiclient.destroyRouter(cmd) + self.routers = [] - def start_router(self): - self.logger.debug('Starting router') + def start_routers(self): + self.logger.debug('Starting routers') for router in self.routers: cmd = startRouter.startRouterCmd() cmd.id = router.id diff --git a/test/integration/component/test_routers_network_ops.py b/test/integration/component/test_routers_network_ops.py index 9937a4e3db1..7f1e4aa4434 100644 --- a/test/integration/component/test_routers_network_ops.py +++ b/test/integration/component/test_routers_network_ops.py @@ -110,6 +110,11 @@ class TestCreatePFOnStoppedRouter(cloudstackTestCase): raise Exception("Warning: Exception during cleanup : %s" % e) return + def setUp(self): + self.apiclient = self.testClient.getApiClient() + self.cleanup = [] + return + def tearDown(self): try: # Clean up, terminate the created resources @@ -118,10 +123,6 @@ class TestCreatePFOnStoppedRouter(cloudstackTestCase): raise Exception("Warning: Exception during cleanup : %s" % e) return - def setUp(self): - self.apiclient = self.testClient.getApiClient() - self.cleanup = [] - return @attr(tags=["advanced", "advancedns"], required_hardware="true") def test_01_CreatePFOnStoppedRouter(self): @@ -322,6 +323,11 @@ class TestCreateLBOnStoppedRouter(cloudstackTestCase): raise Exception("Warning: Exception during cleanup : %s" % e) return + def setUp(self): + self.apiclient = self.testClient.getApiClient() + self.cleanup = [] + return + def tearDown(self): try: cleanup_resources(self.apiclient, self.cleanup) @@ -329,11 +335,6 @@ class TestCreateLBOnStoppedRouter(cloudstackTestCase): raise Exception("Warning: Exception during cleanup : %s" % e) return - def setUp(self): - self.apiclient = self.testClient.getApiClient() - self.cleanup = [] - return - @attr(tags=["advanced", "advancedns"], required_hardware="true") def test_01_CreateLBOnStoppedRouter(self): """Stop existing Router, add LB rule and check we can reach the VM""" diff --git a/test/integration/component/test_vpc_redundant.py b/test/integration/component/test_vpc_redundant.py index e1a6ecb6de0..11fabf697c8 100644 --- a/test/integration/component/test_vpc_redundant.py +++ b/test/integration/component/test_vpc_redundant.py @@ -22,6 +22,7 @@ from nose.plugins.attrib import attr from marvin.cloudstackTestCase import cloudstackTestCase from marvin.lib.base import (stopRouter, startRouter, + destroyRouter, Account, VpcOffering, VPC, @@ -242,7 +243,6 @@ class TestVPCRedundancy(cloudstackTestCase): admin=True, domainid=self.domain.id) - self._cleanup = [self.account] self.logger.debug("Creating a VPC offering..") self.vpc_off = VpcOffering.create( self.apiclient, @@ -260,6 +260,17 @@ class TestVPCRedundancy(cloudstackTestCase): zoneid=self.zone.id, account=self.account.name, domainid=self.account.domainid) + + self.cleanup = [self.vpc, self.vpc_off, self.account] + return + + def tearDown(self): + try: + #Stop/Destroy the routers so we are able to remove the networks. Issue CLOUDSTACK-8935 + self.destroy_routers() + cleanup_resources(self.api_client, self.cleanup) + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) return def query_routers(self, count=2, showall=False): @@ -287,16 +298,29 @@ class TestVPCRedundancy(cloudstackTestCase): if cnts[vals.index('MASTER')] != 1: self.fail("No Master or too many master routers found %s" % cnts[vals.index('MASTER')]) - def stop_router(self, type): + def stop_router(self, router): + self.logger.debug('Stopping router %s' % router.id) + cmd = stopRouter.stopRouterCmd() + cmd.id = router.id + self.apiclient.stopRouter(cmd) + + def stop_router_by_type(self, type): self.check_master_status(2) self.logger.debug('Stopping %s router' % type) for router in self.routers: if router.redundantstate == type: - cmd = stopRouter.stopRouterCmd() - cmd.id = router.id - self.apiclient.stopRouter(cmd) + self.stop_router(router) - def start_router(self): + def destroy_routers(self): + self.logger.debug('Destroying routers') + for router in self.routers: + self.stop_router(router) + cmd = destroyRouter.destroyRouterCmd() + cmd.id = router.id + self.apiclient.destroyRouter(cmd) + self.routers = [] + + def start_routers(self): self.check_master_status(2, showall=True) self.logger.debug('Starting stopped routers') for router in self.routers: @@ -317,6 +341,7 @@ class TestVPCRedundancy(cloudstackTestCase): conservemode=False) nw_off.update(self.apiclient, state='Enabled') + self.logger.debug('Created and Enabled NetworkOffering') self.services["network"]["name"] = "NETWORK-" + str(gateway) @@ -336,8 +361,14 @@ class TestVPCRedundancy(cloudstackTestCase): except Exception, e: self.fail('Unable to create a Network with offering=%s because of %s ' % (net_offerring, e)) o = networkO(obj_network) - o.add_vm(self.deployvm_in_network(obj_network)) - o.add_vm(self.deployvm_in_network(obj_network)) + + vm1 = self.deployvm_in_network(obj_network) + vm2 = self.deployvm_in_network(obj_network) + self.cleanup.insert(2, obj_network) + self.cleanup.insert(3, nw_off) + + o.add_vm(vm1) + o.add_vm(vm2) return o def deployvm_in_network(self, network, host_id=None): @@ -352,7 +383,9 @@ class TestVPCRedundancy(cloudstackTestCase): networkids=[str(network.id)], hostid=host_id ) + self.logger.debug('Created VM=%s in network=%s' % (vm.id, network.name)) + self.cleanup.insert(0, vm) return vm except: self.fail('Unable to create VM in a Network=%s' % network.name) @@ -394,9 +427,10 @@ class TestVPCRedundancy(cloudstackTestCase): traffictype='Ingress' ) self.logger.debug('nwacl_nat=%s' % nwacl_nat.__dict__) + return nat_rule - def check_ssh_into_vm(self, vm, public_ip, expectFail=False, retries=20): + def check_ssh_into_vm(self, vm, public_ip, expectFail=False, retries=5): self.logger.debug("Checking if we can SSH into VM=%s on public_ip=%s (%r)" % (vm.name, public_ip.ipaddress.ipaddress, expectFail)) vm.ssh_client = None @@ -429,7 +463,7 @@ class TestVPCRedundancy(cloudstackTestCase): self.add_nat_rules() self.do_vpc_test(False) - self.stop_router("MASTER") + self.stop_router_by_type("MASTER") # wait for the backup router to transit to master state time.sleep(30) self.check_master_status(1) @@ -440,7 +474,7 @@ class TestVPCRedundancy(cloudstackTestCase): self.check_master_status(1) self.do_vpc_test(True) - self.start_router() + self.start_routers() self.add_nat_rules() time.sleep(45) self.check_master_status(2) @@ -456,8 +490,7 @@ class TestVPCRedundancy(cloudstackTestCase): time.sleep(30) self.check_master_status(2) self.add_nat_rules() - self.test_default_routes() - + self.do_default_routes_test() def delete_nat_rules(self): for o in self.networks: @@ -483,7 +516,7 @@ class TestVPCRedundancy(cloudstackTestCase): for vm in o.get_vms(): self.check_ssh_into_vm(vm.get_vm(), vm.get_ip(), expectFail=expectFail, retries=retries) - def test_default_routes(self): + def do_default_routes_test(self): for o in self.networks: for vmObj in o.get_vms(): ssh_command = "ping -c 3 8.8.8.8" @@ -500,7 +533,7 @@ class TestVPCRedundancy(cloudstackTestCase): self.logger.debug("Ping to google.com from VM") result = str(ssh.execute(ssh_command)) - self.logger.debug("SSH result: %s; COUNT is ==> %s" % (result, result.count("0% packet loss"))) + self.logger.debug("SSH result: %s; COUNT is ==> %s" % (result, result.count("3 packets received"))) except Exception as e: self.fail("SSH Access failed for %s: %s" % \ (vmObj.get_ip(), e) diff --git a/test/integration/component/test_vpc_router_nics.py b/test/integration/component/test_vpc_router_nics.py index db4e27a0e51..cb0734d9930 100644 --- a/test/integration/component/test_vpc_router_nics.py +++ b/test/integration/component/test_vpc_router_nics.py @@ -224,7 +224,6 @@ class TestVPCNics(cloudstackTestCase): admin=True, domainid=self.domain.id) - self.cleanup = [self.account] self.logger.debug("Creating a VPC offering..") self.vpc_off = VpcOffering.create( self.apiclient, @@ -242,10 +241,13 @@ class TestVPCNics(cloudstackTestCase): zoneid=self.zone.id, account=self.account.name, domainid=self.account.domainid) + + self.cleanup = [self.vpc, self.vpc_off, self.account] return def tearDown(self): try: + self.destroy_routers() cleanup_resources(self.apiclient, self.cleanup) except Exception as e: self.logger.debug("Warning: Exception during cleanup : %s" % e) @@ -261,19 +263,20 @@ class TestVPCNics(cloudstackTestCase): isinstance(self.routers, list), True, "Check for list routers response return valid data") - def stop_router(self): + def stop_router(self, router): self.logger.debug('Stopping router') - for router in self.routers: - cmd = stopRouter.stopRouterCmd() - cmd.id = router.id - self.apiclient.stopRouter(cmd) + cmd = stopRouter.stopRouterCmd() + cmd.id = router.id + self.apiclient.stopRouter(cmd) - def destroy_router(self): - self.logger.debug('Stopping router') + def destroy_routers(self): + self.logger.debug('Destroying routers') for router in self.routers: + self.stop_router(router) cmd = destroyRouter.destroyRouterCmd() cmd.id = router.id self.apiclient.destroyRouter(cmd) + self.routers = [] def create_network(self, net_offerring, gateway='10.1.1.1', vpc=None): try: @@ -299,12 +302,19 @@ class TestVPCNics(cloudstackTestCase): gateway=gateway, vpcid=vpc.id if vpc else self.vpc.id ) - + + self.logger.debug("Created network with ID: %s" % obj_network.id) except Exception, e: self.fail('Unable to create a Network with offering=%s because of %s ' % (net_offerring, e)) o = networkO(obj_network) - o.add_vm(self.deployvm_in_network(obj_network)) + + vm1 = self.deployvm_in_network(obj_network) + + self.cleanup.insert(1, obj_network) + self.cleanup.insert(2, nw_off) + + o.add_vm(vm1) return o def deployvm_in_network(self, network): @@ -318,7 +328,9 @@ class TestVPCNics(cloudstackTestCase): serviceofferingid=self.service_offering.id, networkids=[str(network.id)] ) + self.logger.debug('Created VM=%s in network=%s' % (vm.id, network.name)) + self.cleanup.insert(0, vm) return vm except: self.fail('Unable to create VM in a Network=%s' % network.name) @@ -362,17 +374,6 @@ class TestVPCNics(cloudstackTestCase): self.logger.debug('nwacl_nat=%s' % nwacl_nat.__dict__) return nat_rule - def check_ssh_into_vm(self, vm, public_ip): - self.logger.debug("Checking if we can SSH into VM=%s on public_ip=%s" % - (vm.name, public_ip.ipaddress.ipaddress)) - vm.ssh_client = None - try: - vm.get_ssh_client(ipaddress=public_ip.ipaddress.ipaddress) - self.logger.debug("SSH into VM=%s on public_ip=%s is successful" % - (vm.name, public_ip.ipaddress.ipaddress)) - except: - self.fail("Failed to SSH into VM - %s" % (public_ip.ipaddress.ipaddress)) - @attr(tags=["advanced", "intervlan"], required_hardware="true") def test_01_VPC_nics_after_destroy(self): """ Create a VPC with two networks with one VM in each network and test nics after destroy""" @@ -386,15 +387,16 @@ class TestVPCNics(cloudstackTestCase): self.networks.append(net2) self.add_nat_rules() - self.test_ssh_to_vm() + self.check_ssh_into_vm() - self.stop_router() - self.destroy_router() + self.destroy_routers() time.sleep(30) net1.add_vm(self.deployvm_in_network(net1.get_net())) + self.query_routers() + self.add_nat_rules() - self.test_ssh_to_vm() + self.check_ssh_into_vm() @attr(tags=["advanced", "intervlan"], required_hardware="true") def test_02_VPC_default_routes(self): @@ -409,7 +411,7 @@ class TestVPCNics(cloudstackTestCase): self.networks.append(net2) self.add_nat_rules() - self.test_default_routes() + self.do_default_routes_test() def delete_nat_rules(self): for o in self.networks: @@ -427,12 +429,25 @@ class TestVPCNics(cloudstackTestCase): vm.set_nat(self.create_natrule(vm.get_vm(), vm.get_ip(), o.get_net())) time.sleep(5) - def test_ssh_to_vm(self): + def check_ssh_into_vm(self): for o in self.networks: for vm in o.get_vms(): - self.check_ssh_into_vm(vm.get_vm(), vm.get_ip()) + try: + virtual_machine = vm.get_vm() + virtual_machine.ssh_client = None - def test_default_routes(self): + public_ip = vm.get_ip() + + self.logger.debug("Checking if we can SSH into VM=%s on public_ip=%s" % + (virtual_machine.name, public_ip.ipaddress.ipaddress)) + + virtual_machine.get_ssh_client(ipaddress=public_ip.ipaddress.ipaddress) + self.logger.debug("SSH into VM=%s on public_ip=%s is successful" % + (virtual_machine.name, public_ip.ipaddress.ipaddress)) + except: + self.fail("Failed to SSH into VM - %s" % (public_ip.ipaddress.ipaddress)) + + def do_default_routes_test(self): for o in self.networks: for vmObj in o.get_vms(): ssh_command = "ping -c 3 8.8.8.8" @@ -449,7 +464,7 @@ class TestVPCNics(cloudstackTestCase): self.logger.debug("Ping to google.com from VM") result = str(ssh.execute(ssh_command)) - self.logger.debug("SSH result: %s; COUNT is ==> %s" % (result, result.count("0% packet loss"))) + self.logger.debug("SSH result: %s; COUNT is ==> %s" % (result, result.count("3 packets received"))) except Exception as e: self.fail("SSH Access failed for %s: %s" % \ (vmObj.get_ip(), e)