CLOUDSTACK-4499: Wait until hosts are up before adding storage pools

This works around the delay caused in adding Xen 6.1/6.2 hosts where
host takes two attempts to transition to Up state. We will wait for one
minute per cluster before attempting to add storage pool in the cluster.

Signed-off-by: Prasanna Santhanam <tsp@apache.org>
(cherry picked from commit 22ee499c3571b2a6b6921abb36c679128893c415)

tools/marvin/marvin/deployDataCenter.py

@@ -22,6 +22,7 @@ import cloudstackTestClient
 import logging
 from cloudstackAPI import *
 from os import path
+from time import sleep
 from optparse import OptionParser
 
 
@@ -87,9 +88,28 @@ specify a valid config file" % cfgFile)
             if cluster.hypervisor.lower() != "vmware":
                 self.addHosts(cluster.hosts, zoneId, podId, clusterId,
                               cluster.hypervisor)
+            self.wait_for_host(zoneId, clusterId)
             self.createPrimaryStorages(cluster.primaryStorages, zoneId, podId,
                                        clusterId)
 
+    def wait_for_host(self, zoneId, clusterId):
+        """
+        Wait for the hosts in the zoneid, clusterid to be up
+
+        2 retries with 30s delay
+        """
+        retry, timeout = 2, 30
+        cmd = listHosts.listHostsCmd()
+        cmd.clusterid, cmd.zoneid = clusterId, zoneId
+        hosts = self.apiClient.listHosts(cmd)
+        while retry != 0:
+            for host in hosts:
+                if host.state != 'Up':
+                    break
+            sleep(timeout)
+            retry = retry - 1
+
+
     def createPrimaryStorages(self, primaryStorages, zoneId, podId, clusterId):
         if primaryStorages is None:
             return