From 52bbf4a185131a8aeb6df800cfeae5ed6d3ac6fa Mon Sep 17 00:00:00 2001 From: Chirag Jog Date: Mon, 6 Aug 2012 12:35:51 +0530 Subject: [PATCH] Dedicated Host HA Tests --- .../component/test_host_high_availability.py | 796 ++++++++++++++++++ test/integration/lib/common.py | 21 + 2 files changed, 817 insertions(+) create mode 100644 test/integration/component/test_host_high_availability.py diff --git a/test/integration/component/test_host_high_availability.py b/test/integration/component/test_host_high_availability.py new file mode 100644 index 00000000000..a019c0970f4 --- /dev/null +++ b/test/integration/component/test_host_high_availability.py @@ -0,0 +1,796 @@ +# -*- encoding: utf-8 -*- +# Copyright 2012 Citrix Systems, Inc. Licensed under the +# Apache License, Version 2.0 (the "License"); you may not use this +# file except in compliance with the License. Citrix Systems, Inc. +# reserves all rights not expressly granted by the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Automatically generated by addcopyright.py at 04/03/2012 + +""" P1 tests for dedicated Host high availability +""" +#Import Local Modules +import marvin +from marvin.cloudstackTestCase import * +from marvin.cloudstackAPI import * +from integration.lib.utils import * +from integration.lib.base import * +from integration.lib.common import * +from marvin import remoteSSHClient +import datetime + + +class Services: + """ Dedicated host HA test cases """ + + def __init__(self): + self.services = { + "account": { + "email": "test@test.com", + "firstname": "HA", + "lastname": "HA", + "username": "HA", + # Random characters are appended for unique + # username + "password": "password", + }, + "service_offering_with_ha": { + "name": "Tiny Instance With HA Enabled", + "displaytext": "Tiny Instance", + "cpunumber": 1, + "cpuspeed": 100, # in MHz + "memory": 64, # In MBs + }, + "service_offering_without_ha": { + "name": "Tiny Instance Without HA", + "displaytext": "Tiny Instance", + "cpunumber": 1, + "cpuspeed": 100, # in MHz + "memory": 64, # In MBs + }, + "virtual_machine": { + "displayname": "VM", + "username": "root", + "password": "password", + "ssh_port": 22, + "hypervisor": 'XenServer', + # Hypervisor type should be same as + # hypervisor type of cluster + "privateport": 22, + "publicport": 22, + "protocol": 'TCP', + }, + "ostypeid": '6a6b23aa-5e59-4b53-8161-0ad1aa5990f1', + "timeout": 100, + } + +class TestHostHighAvailability(cloudstackTestCase): + """ Dedicated host HA test cases """ + + @classmethod + def setUpClass(cls): + + cls.api_client = super( + TestHostHighAvailability, + cls + ).getClsTestClient().getApiClient() + cls.services = Services().services + # Get Zone, Domain and templates + cls.domain = get_domain( + cls.api_client, + cls.services + ) + cls.zone = get_zone( + cls.api_client, + cls.services + ) + + cls.template = get_template( + cls.api_client, + cls.zone.id, + cls.services["ostypeid"] + ) + cls.services["virtual_machine"]["zoneid"] = cls.zone.id + cls.services["virtual_machine"]["template"] = cls.template.id + + cls.service_offering_with_ha = ServiceOffering.create( + cls.api_client, + cls.services["service_offering_with_ha"], + offerha=True + ) + + cls.service_offering_without_ha = ServiceOffering.create( + cls.api_client, + cls.services["service_offering_without_ha"], + offerha=False + ) + + cls._cleanup = [ + cls.service_offering_with_ha, + cls.service_offering_without_ha, + ] + return + + @classmethod + def tearDownClass(cls): + try: + #Cleanup resources used + cleanup_resources(cls.api_client, cls._cleanup) + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) + return + + def setUp(self): + self.apiclient = self.testClient.getApiClient() + self.dbclient = self.testClient.getDbConnection() + self.account = Account.create( + self.apiclient, + self.services["account"], + admin=True, + domainid=self.domain.id + ) + self.cleanup = [self.account] + return + + def tearDown(self): + try: + #Clean up, terminate the created accounts, domains etc + cleanup_resources(self.apiclient, self.cleanup) + self.testClient.close() + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) + return + + def test_01_vm_deployment_with_compute_offering_with_ha_enabled(self): + """ Test VM deployments (Create HA enabled Compute Service Offering and VM) """ + + # Steps, + #1. Create a Compute service offering with the “Offer HA” option selected. + #2. Create a Guest VM with the compute service offering created above. + # Validations, + #1. Ensure that the offering is created and that in the UI the “Offer HA” field is enabled (Yes) + #The listServiceOffering API should list “offerha” as true. + #2. Select the newly created VM and ensure that the Compute offering field value lists the compute service offering that was selected. + # Also, check that the HA Enabled field is enabled “Yes”. + + #list and validate above created service offering with Ha enabled + list_service_response = list_service_offering( + self.apiclient, + id=self.service_offering_with_ha.id + ) + self.assertEqual( + isinstance(list_service_response, list), + True, + "listServiceOfferings returned invalid object in response." + ) + self.assertNotEqual( + len(list_service_response), + 0, + "listServiceOfferings returned empty list." + ) + self.assertEqual( + list_service_response[0].offerha, + True, + "The service offering is not HA enabled" + ) + + #create virtual machine with the service offering with Ha enabled + virtual_machine = VirtualMachine.create( + self.apiclient, + self.services["virtual_machine"], + accountid=self.account.account.name, + domainid=self.account.account.domainid, + serviceofferingid=self.service_offering_with_ha.id + ) + vms = VirtualMachine.list( + self.apiclient, + id=virtual_machine.id, + listall=True + ) + self.assertEqual( + isinstance(vms, list), + True, + "listVirtualMachines returned invalid object in response." + ) + self.assertNotEqual( + len(vms), + 0, + "listVirtualMachines returned empty list." + ) + self.debug("Deployed VM on host: %s" % vms[0].hostid) + self.assertEqual( + vms[0].haenable, + True, + "VM not created with HA enable tag" + ) + + def test_02_no_vm_creation_on_host_with_haenabled(self): + """ Verify you can not create new VMs on hosts with an ha.tag """ + + # Steps, + #1. Fresh install CS (Bonita) that supports this feature + #2. Create Basic zone, pod, cluster, add 3 hosts to cluster (host1, host2, host3), secondary & primary Storage + #3. When adding host3, assign the HA host tag. + #4. You should already have a compute service offering with HA already create from above. If not, create one for HA. + #5. Create VMs with the service offering with and without the HA tag + # Validations, + #Check to make sure the newly created VM is not on any HA enabled hosts + #The VM should be created only on host1 or host2 and never host3 (HA enabled) + + #create and verify virtual machine with HA enabled service offering + virtual_machine_with_ha = VirtualMachine.create( + self.apiclient, + self.services["virtual_machine"], + accountid=self.account.account.name, + domainid=self.account.account.domainid, + serviceofferingid=self.service_offering_with_ha.id + ) + + vms = VirtualMachine.list( + self.apiclient, + id=virtual_machine_with_ha.id, + listall=True + ) + + self.assertEqual( + isinstance(vms, list), + True, + "listVirtualMachines returned invalid object in response." + ) + + self.assertNotEqual( + len(vms), + 0, + "listVirtualMachines returned empty list." + ) + + vm = vms[0] + + self.debug("Deployed VM on host: %s" % vm.hostid) + + #validate the virtual machine created is host Ha enabled + list_hosts_response = list_hosts( + self.apiclient, + id=vm.hostid + ) + self.assertEqual( + isinstance(list_hosts_response, list), + True, + "listHosts returned invalid object in response." + ) + + self.assertNotEqual( + len(list_hosts_response), + 0, + "listHosts retuned empty list in response." + ) + + self.assertEqual( + list_hosts_response[0].hahost, + False, + "VM created on HA enabled host." + ) + + #create and verify virtual machine with Ha disabled service offering + virtual_machine_without_ha = VirtualMachine.create( + self.apiclient, + self.services["virtual_machine"], + accountid=self.account.account.name, + domainid=self.account.account.domainid, + serviceofferingid=self.service_offering_without_ha.id + ) + + vms = VirtualMachine.list( + self.apiclient, + id=virtual_machine_without_ha.id, + listall=True + ) + + self.assertEqual( + isinstance(vms, list), + True, + "listVirtualMachines returned invalid object in response." + ) + + self.assertNotEqual( + len(vms), + 0, + "listVirtualMachines returned empty list." + ) + + vm = vms[0] + + self.debug("Deployed VM on host: %s" % vm.hostid) + + #verify that the virtual machine created on the host is Ha disabled + list_hosts_response = list_hosts( + self.apiclient, + id=vm.hostid + ) + self.assertEqual( + isinstance(list_hosts_response, list), + True, + "listHosts returned invalid object in response." + ) + + self.assertNotEqual( + len(list_hosts_response), + 0, + "listHosts returned empty list." + ) + + host = list_hosts_response[0] + + self.assertEqual( + host.hahost, + False, + "VM migrated to HA enabled host." + ) + + def test_03_cant_migrate_vm_to_host_with_ha_positive(self): + """ Verify you can not migrate VMs to hosts with an ha.tag (positive) """ + + # Steps, + #1. Create a Compute service offering with the “Offer HA” option selected. + #2. Create a Guest VM with the compute service offering created above. + #3. Select the VM and migrate VM to another host. Choose a “Suitable” host (i.e. host2) + # Validations + #The option from the “Migrate instance to another host” dialog box” should list host3 as “Not Suitable” for migration. + #Confirm that the VM is migrated to the “Suitable” host you selected (i.e. host2) + + #create and verify the virtual machine with HA enabled service offering + virtual_machine_with_ha = VirtualMachine.create( + self.apiclient, + self.services["virtual_machine"], + accountid=self.account.account.name, + domainid=self.account.account.domainid, + serviceofferingid=self.service_offering_with_ha.id + ) + + vms = VirtualMachine.list( + self.apiclient, + id=virtual_machine_with_ha.id, + listall=True, + ) + + self.assertEqual( + isinstance(vms, list), + True, + "List VMs should return valid response for deployed VM" + ) + + self.assertNotEqual( + len(vms), + 0, + "List VMs should return valid response for deployed VM" + ) + + vm = vms[0] + + self.debug("Deployed VM on host: %s" % vm.hostid) + + #Find out a Suitable host for VM migration + list_hosts_response = list_hosts( + self.apiclient, + ) + self.assertEqual( + isinstance(list_hosts_response, list), + True, + "The listHosts API returned the invalid list" + ) + + self.assertNotEqual( + len(list_hosts_response), + 0, + "The listHosts returned nothing." + ) + suitableHost = None + for host in list_hosts_response: + if host.suitableformigration == True and host.hostid != vm.hostid: + suitableHost = host + break + + self.assertTrue(suitableHost is not None, "suitablehost should not be None") + + #Migration of the VM to a suitable host + self.debug("Migrating VM-ID: %s to Host: %s" % (self.vm.id, suitableHost.id)) + + cmd = migrateVirtualMachine.migrateVirtualMachineCmd() + cmd.hostid = suitableHost.id + cmd.virtualmachineid = self.vm.id + self.apiclient.migrateVirtualMachine(cmd) + + #Verify that the VM migrated to a targeted Suitable host + list_vm_response = list_virtual_machines( + self.apiclient, + id=vm.id + ) + self.assertEqual( + isinstance(list_vm_response, list), + True, + "The listVirtualMachines returned the invalid list." + ) + + self.assertNotEqual( + list_vm_response, + None, + "The listVirtualMachines API returned nothing." + ) + + vm_response = list_vm_response[0] + + self.assertEqual( + vm_response.id, + vm.id, + "The virtual machine id and the the virtual machine from listVirtualMachines is not matching." + ) + + self.assertEqual( + vm_response.hostid, + suitableHost.id, + "The VM is not migrated to targeted suitable host." + ) + + def test_04_cant_migrate_vm_to_host_with_ha_negative(self): + """ Verify you can not migrate VMs to hosts with an ha.tag (negative) """ + + # Steps, + #1. Create a Compute service offering with the “Offer HA” option selected. + #2. Create a Guest VM with the compute service offering created above. + #3. Select the VM and migrate VM to another host. Choose a “Not Suitable” host. + # Validations, + #The option from the “Migrate instance to another host” dialog box” should list host3 as “Not Suitable” for migration. + #By design, The Guest VM can STILL can be migrated to host3 if the admin chooses to do so. + + #create and verify virtual machine with HA enabled service offering + virtual_machine_with_ha = VirtualMachine.create( + self.apiclient, + self.services["virtual_machine"], + accountid=self.account.account.name, + domainid=self.account.account.domainid, + serviceofferingid=self.service_offering_with_ha.id + ) + + vms = VirtualMachine.list( + self.apiclient, + id=virtual_machine_with_ha.id, + listall=True + ) + + self.assertEqual( + isinstance(vms, list), + True, + "The listVirtualMachines returned invalid object in response." + ) + + self.assertNotEqual( + len(vms), + 0, + "The listVirtualMachines returned empty response." + ) + + vm = vms[0] + + self.debug("Deployed VM on host: %s" % vm.hostid) + + #Find out Non-Suitable host for VM migration + list_hosts_response = list_hosts( + self.apiclient, + ) + self.assertEqual( + isinstance(list_hosts_response, list), + True, + "listHosts returned invalid object in response." + ) + + self.assertNotEqual( + len(list_hosts_response), + 0, + "listHosts returned empty response." + ) + + notSuitableHost = None + for host in list_hosts_response: + if not host.suitableformigration and host.hostid != vm.hostid: + notSuitableHost = host + break + + self.assertTrue(notSuitableHost is not None, "notsuitablehost should not be None") + + #Migrate VM to Non-Suitable host + self.debug("Migrating VM-ID: %s to Host: %s" % (vm.id, notSuitableHost.id)) + + cmd = migrateVirtualMachine.migrateVirtualMachineCmd() + cmd.hostid = notSuitableHost.id + cmd.virtualmachineid = vm.id + self.apiclient.migrateVirtualMachine(cmd) + + #Verify that the virtual machine got migrated to targeted Non-Suitable host + list_vm_response = list_virtual_machines( + self.apiclient, + id=vm.id + ) + self.assertEqual( + isinstance(list_vm_response, list), + True, + "listVirtualMachine returned invalid object in response." + ) + + self.assertNotEqual( + len(list_vm_response), + 0, + "listVirtualMachines returned empty response." + ) + + self.assertEqual( + list_vm_response[0].id, + vm.id, + "Virtual machine id with the virtual machine from listVirtualMachine is not matching." + ) + + self.assertEqual( + list_vm_response[0].hostid, + notSuitableHost.id, + "The detination host id of migrated VM is not matching." + ) + + def test_05_no_vm_with_ha_gets_migrated_to_ha_host_in_live_migration(self): + """ Verify that none of the VMs with HA enabled migrate to an ha tagged host during live migration """ + + # Steps, + #1. Fresh install CS (Bonita) that supports this feature + #2. Create Basic zone, pod, cluster, add 3 hosts to cluster (host1, host2, host3), secondary & primary Storage + #3. When adding host3, assign the HA host tag. + #4. Create VMs with and without the Compute Service Offering with the HA tag. + #5. Note the VMs on host1 and whether any of the VMs have their “HA enabled” flags enabled. + #6. Put host1 into maintenance mode. + # Validations, + #1. Make sure the VMs are created on either host1 or host2 and not on host3 + #2. Putting host1 into maintenance mode should trigger a live migration. Make sure the VMs are not migrated to HA enabled host3. + + # create and verify virtual machine with HA disabled service offering + virtual_machine_with_ha = VirtualMachine.create( + self.apiclient, + self.services["virtual_machine"], + accountid=self.account.account.name, + domainid=self.account.account.domainid, + serviceofferingid=self.service_offering_with_ha.id + ) + + vms = VirtualMachine.list( + self.apiclient, + id=virtual_machine_with_ha.id, + listall=True + ) + + self.assertEqual( + isinstance(vms, list), + True, + "List VMs should return valid response for deployed VM" + ) + + self.assertNotEqual( + len(vms), + 0, + "List VMs should return valid response for deployed VM" + ) + + vm_with_ha_enabled = vms[0] + + #Verify the virtual machine got created on non HA host + list_hosts_response = list_hosts( + self.apiclient, + id=vm_with_ha_enabled.hostid + ) + self.assertEqual( + isinstance(list_hosts_response, list), + True, + "Check list response returns a valid list" + ) + + self.assertNotEqual( + len(list_hosts_response), + 0, + "Check Host is available" + ) + + self.assertEqual( + list_hosts_response[0].hahost, + False, + "The virtual machine is not ha enabled so check if VM is created on host which is also not ha enabled" + ) + + #put the Host in maintainance mode + self.debug("Enabling maintenance mode for host %s" % vm_with_ha_enabled.hostid) + cmd = prepareHostForMaintenance.prepareHostForMaintenanceCmd() + cmd.id = vm_with_ha_enabled.hostid + self.apiclient.prepareHostForMaintenance(cmd) + + timeout = self.services["timeout"] + + #verify the VM live migration happened to another running host + self.debug("Waiting for VM to come up") + wait_for_vm( + self.apiclient, + virtualmachineid=vm_with_ha_enabled.id, + interval=timeout + ) + + vms = VirtualMachine.list( + self.apiclient, + id=vm_with_ha_enabled.id, + listall=True, + ) + + self.assertEqual( + isinstance(vms, list), + True, + "List VMs should return valid response for deployed VM" + ) + + self.assertNotEqual( + len(vms), + 0, + "List VMs should return valid response for deployed VM" + ) + + vm_with_ha_enabled1 = vms[0] + + list_hosts_response = list_hosts( + self.apiclient, + id=vm_with_ha_enabled1.hostid + ) + self.assertEqual( + isinstance(list_hosts_response, list), + True, + "Check list response returns a valid list" + ) + + self.assertNotEqual( + len(list_hosts_response), + 0, + "Check Host is available" + ) + + self.assertEqual( + list_hosts_response[0].hahost, + False, + "The virtual machine is not ha enabled so check if VM is created on host which is also not ha enabled" + ) + + self.debug("Disabling the maintenance mode for host %s" % vm_with_ha_enabled.hostid) + cmd = cancelHostMaintenance.cancelHostMaintenanceCmd() + cmd.id = vm_with_ha_enabled.hostid + self.apiclient.cancelHostMaintenance(cmd) + + def test_06_no_vm_without_ha_gets_migrated_to_ha_host_in_live_migration(self): + """ Verify that none of the VMs without HA enabled migrate to an ha tagged host during live migration """ + + # Steps, + #1. Fresh install CS (Bonita) that supports this feature + #2. Create Basic zone, pod, cluster, add 3 hosts to cluster (host1, host2, host3), secondary & primary Storage + #3. When adding host3, assign the HA host tag. + #4. Create VMs with and without the Compute Service Offering with the HA tag. + #5. Note the VMs on host1 and whether any of the VMs have their “HA enabled” flags enabled. + #6. Put host1 into maintenance mode. + # Validations, + #1. Make sure the VMs are created on either host1 or host2 and not on host3 + #2. Putting host1 into maintenance mode should trigger a live migration. Make sure the VMs are not migrated to HA enabled host3. + + # create and verify virtual machine with HA disabled service offering + virtual_machine_without_ha = VirtualMachine.create( + self.apiclient, + self.services["virtual_machine"], + accountid=self.account.account.name, + domainid=self.account.account.domainid, + serviceofferingid=self.service_offering_without_ha.id + ) + + vms = VirtualMachine.list( + self.apiclient, + id=virtual_machine_without_ha.id, + listall=True + ) + + self.assertEqual( + isinstance(vms, list), + True, + "List VMs should return valid response for deployed VM" + ) + + self.assertNotEqual( + len(vms), + 0, + "List VMs should return valid response for deployed VM" + ) + + vm_with_ha_disabled = vms[0] + + #Verify the virtual machine got created on non HA host + list_hosts_response = list_hosts( + self.apiclient, + id=vm_with_ha_disabled.hostid + ) + self.assertEqual( + isinstance(list_hosts_response, list), + True, + "Check list response returns a valid list" + ) + + self.assertNotEqual( + len(list_hosts_response), + 0, + "Check Host is available" + ) + + self.assertEqual( + list_hosts_response[0].hahost, + False, + "The virtual machine is not ha enabled so check if VM is created on host which is also not ha enabled" + ) + + #put the Host in maintainance mode + self.debug("Enabling maintenance mode for host %s" % vm_with_ha_disabled.hostid) + cmd = prepareHostForMaintenance.prepareHostForMaintenanceCmd() + cmd.id = vm_with_ha_disabled.hostid + self.apiclient.prepareHostForMaintenance(cmd) + + timeout = self.services["timeout"] + + #verify the VM live migration happened to another running host + self.debug("Waiting for VM to come up") + wait_for_vm( + self.apiclient, + virtualmachineid=vm_with_ha_disabled.id, + interval=timeout + ) + + vms = VirtualMachine.list( + self.apiclient, + id=vm_with_ha_disabled.id, + listall=True + ) + + self.assertEqual( + isinstance(vms, list), + True, + "List VMs should return valid response for deployed VM" + ) + + self.assertNotEqual( + len(vms), + 0, + "List VMs should return valid response for deployed VM" + ) + + list_hosts_response = list_hosts( + self.apiclient, + id=vms[0].hostid + ) + self.assertEqual( + isinstance(list_hosts_response, list), + True, + "Check list response returns a valid list" + ) + + self.assertNotEqual( + len(list_hosts_response), + 0, + "Check Host is available" + ) + + self.assertEqual( + list_hosts_response[0].hahost, + False, + "The virtual machine is not ha enabled so check if VM is created on host which is also not ha enabled" + ) + + self.debug("Disabling the maintenance mode for host %s" % vm_with_ha_disabled.hostid) + cmd = cancelHostMaintenance.cancelHostMaintenanceCmd() + cmd.id = vm_with_ha_disabled.hostid + self.apiclient.cancelHostMaintenance(cmd) diff --git a/test/integration/lib/common.py b/test/integration/lib/common.py index 5a644aaf16a..630971c606f 100644 --- a/test/integration/lib/common.py +++ b/test/integration/lib/common.py @@ -185,6 +185,27 @@ def wait_for_ssvms(apiclient, zoneid, podid, interval=60): break return +def wait_for_vm(apiclient, virtualmachineid, interval=60): + """After setup wait for VM to come Up""" + + time.sleep(interval) + timeout = 40 + while True: + list_vm_response = list_virtual_machines( + apiclient, + id=virtualmachineid + ) + vm = list_vm_response[0] + if vm.state != 'Running': + # Sleep to ensure VM is Up and Running + time.sleep(interval) + timeout = timeout - 1 + elif vm.state == 'Running': + break + elif timeout == 0: + raise Exception("VM failed to come up") + break + return def download_builtin_templates(apiclient, zoneid, hypervisor, host, linklocalip, interval=60):