test: additional check to ensure hosts are left in up state (#8383)

With this change, a fix is added for failures seen with test_08_migrate_vm or other migration-related tests because the target host is in `Connecting` state,
#8356 (comment)
#8374 (comment)
and more

test/integration/smoke/test_vm_life_cycle.py

@@ -1011,8 +1011,37 @@ class TestSecuredVmMigration(cloudstackTestCase):
 
     @classmethod
     def tearDownClass(cls):
+        if cls.hypervisor.lower() in ["kvm"]:
+            cls.ensure_all_hosts_are_up()
         super(TestSecuredVmMigration, cls).tearDownClass()
 
+    @classmethod
+    def ensure_all_hosts_are_up(cls):
+        hosts = Host.list(
+            cls.apiclient,
+            zoneid=cls.zone.id,
+            type='Routing',
+            hypervisor='KVM'
+        )
+        for host in hosts:
+            if host.state != "Up":
+                SshClient(host.ipaddress, port=22, user=cls.hostConfig["username"], passwd=cls.hostConfig["password"]) \
+                    .execute("service cloudstack-agent stop ; \
+                              sleep 10 ; \
+                              service cloudstack-agent start")
+                interval = 5
+                retries = 10
+                while retries > -1:
+                    time.sleep(interval)
+                    restarted_host = Host.list(
+                        cls.apiclient,
+                        hostid=host.id,
+                        type='Routing'
+                    )[0]
+                    if restarted_host.state == "Up":
+                        break
+                    retries = retries - 1
+
     def setUp(self):
         self.apiclient = self.testClient.getApiClient()
         self.dbclient = self.testClient.getDbConnection()