From d06f2b6a70f4ed399f6edddef82b7cae81d0b71a Mon Sep 17 00:00:00 2001 From: Santhosh Edukulla Date: Mon, 24 Feb 2014 18:55:41 +0530 Subject: [PATCH] CLOUDSTACK-6158: Fix few changes --- tools/marvin/marvin/cloudstackConnection.py | 2 +- tools/marvin/marvin/cloudstackTestClient.py | 2 +- tools/marvin/marvin/deployDataCenter.py | 730 ++++++++++++++------ tools/marvin/marvin/marvinLog.py | 3 +- tools/marvin/marvin/marvinPlugin.py | 43 +- 5 files changed, 564 insertions(+), 216 deletions(-) diff --git a/tools/marvin/marvin/cloudstackConnection.py b/tools/marvin/marvin/cloudstackConnection.py index ee1ff00a907..26116ccb7d8 100644 --- a/tools/marvin/marvin/cloudstackConnection.py +++ b/tools/marvin/marvin/cloudstackConnection.py @@ -244,6 +244,7 @@ class CSConnection(object): cmd_name = '' payload = {} required = [] + isAsync = "false" for attribute in dir(cmd): if not attribute.startswith('__'): if attribute == "isAsync": @@ -274,7 +275,6 @@ class CSConnection(object): for k, v in val.iteritems(): payload["%s[%d].%s" % (param, i, k)] = v i += 1 - return cmd_name.strip(), isAsync, payload except Exception, e: self.logger.\ diff --git a/tools/marvin/marvin/cloudstackTestClient.py b/tools/marvin/marvin/cloudstackTestClient.py index adb7ce365cc..e3edd177535 100644 --- a/tools/marvin/marvin/cloudstackTestClient.py +++ b/tools/marvin/marvin/cloudstackTestClient.py @@ -99,7 +99,7 @@ class CSTestClient(object): if self.__parsedTestDataConfig: ret = self.__parsedTestDataConfig.get("zone") if ret != "NA": - self.__zone = ret + self.__zone = ret return self.__zone def __setHypervisorToClient(self): diff --git a/tools/marvin/marvin/deployDataCenter.py b/tools/marvin/marvin/deployDataCenter.py index bf4b6887c76..d4a0b851520 100644 --- a/tools/marvin/marvin/deployDataCenter.py +++ b/tools/marvin/marvin/deployDataCenter.py @@ -15,31 +15,89 @@ # specific language governing permissions and limitations # under the License. -"""Deploy datacenters according to a json configuration file""" -import configGenerator -from cloudstackException import ( +""" +@Desc : +class DeployDataCenters: Deployes datacenters according to a json + configuration file +class DeleteDataCenter: Deletes a datacenter based upon the dc cfg + settings provided. + This settings file is exported + configuration from DeployDataCenters post + its success +""" +from marvin import configGenerator +from marvin.cloudstackException import ( InvalidParameterException, GetDetailExceptionInfo) - -import logging -from cloudstackAPI import * -from os import path -from time import sleep -from optparse import OptionParser +from marvin.cloudstackAPI import * from marvin.codes import (FAILED, SUCCESS) from marvin.lib.utils import (random_gen) from sys import exit +import os +import pickle +from time import sleep, time +from optparse import OptionParser class DeployDataCenters(object): + ''' @Desc : Deploys the Data Center with information provided ''' - def __init__(self, test_client, cfg, logger=None): - self.testClient = test_client - self.config = cfg - self.tcRunLogger = logger - self.apiClient = None + + def __init__(self, + test_client, + cfg, + logger=None, + log_folder_path=None + ): + self.__testClient = test_client + self.__config = cfg + self.__tcRunLogger = logger + self.__logFolderPath = log_folder_path + self.__apiClient = None + self.__cleanUp = {} + + def __persistDcConfig(self): + if self.__logFolderPath: + dc_file_path = self.__logFolderPath + "/dc_entries.obj" + else: + ts = time.strftime("%b_%d_%Y_%H_%M_%S", + time.localtime()) + dc_file_path = "dc_entries_" + str(ts) + ".obj" + file_to_write = open(dc_file_path, 'w') + if file_to_write: + pickle.dump(self.__cleanUp, file_to_write) + print "\n=== Data Center Settings are dumped to %s" % dc_file_path + self.__tcRunLogger.debug( + "\n=== Data Center Settings are dumped to %s===" % + dc_file_path) + + def __cleanAndExit(self): + print "\n===Deploy DC Failed, So Cleaning to Exit===" + self.__tcRunLogger.debug("===Deploy DC Failed, So Cleaning to Exit===") + remove_dc_obj = DeleteDataCenter(self.__testClient, + dc_cfg=self.__cleanUp, + tc_run_logger=self.__tcRunLogger + ) + if remove_dc_obj: + if remove_dc_obj.removeDataCenter() == FAILED: + print "\n===Removing DataCenter Failed===" + self.__tcRunLogger.debug("===Removing DataCenter Failed===") + else: + print "\n===Removing DataCenter Successful===" + self.__tcRunLogger.debug( + "===Removing DataCenter Successful===") + exit(1) + + def __addToCleanUp(self, type, id): + if type not in self.__cleanUp.keys(): + self.__cleanUp[type] = [] + self.__cleanUp[type].append(id) + if "order" not in self.__cleanUp.keys(): + self.__cleanUp["order"] = [] + if type not in self.__cleanUp["order"]: + self.__cleanUp["order"].append(type) def addHosts(self, hosts, zoneId, podId, clusterId, hypervisor): if hosts is None: @@ -59,9 +117,13 @@ class DeployDataCenters(object): hostcmd.username = host.username hostcmd.zoneid = zoneId hostcmd.hypervisor = hypervisor - if self.apiClient.addHost(hostcmd) == FAILED: - self.tcRunLogger.exception("=== Adding Host Failed===") - exit(1) + ret = self.__apiClient.addHost(hostcmd) + if ret != FAILED and ret[0].id: + self.__tcRunLogger.debug("=== Add Host Successful ===") + self.__addToCleanUp("Host", ret[0].id) + else: + self.__tcRunLogger.exception("=== Adding Host Failed ===") + self.__cleanAndExit() def addVmWareDataCenter(self, vmwareDc): vdc = addVmwareDc.addVmwareDcCmd() @@ -70,9 +132,13 @@ class DeployDataCenters(object): vdc.vcenter = vmwareDc.vcenter vdc.username = vmwareDc.username vdc.password = vmwareDc.password - if self.apiClient.addVmwareDc(vdc) == FAILED: - self.tcRunLogger.exception("=== Adding VmWare DC Failed===") - exit(1) + ret = self.__apiClient.addVmwareDc(vdc) + if ret != FAILED and ret.id: + self.__tcRunLogger.debug("=== Adding VmWare DC Successful===") + self.__addToCleanUp("VmwareDc", ret.id) + else: + self.__tcRunLogger.exception("=== Adding VmWare DC Failed===") + self.__cleanAndExit() def createClusters(self, clusters, zoneId, podId, vmwareDc=None): if clusters is None: @@ -91,16 +157,18 @@ class DeployDataCenters(object): clustercmd.url = cluster.url clustercmd.username = cluster.username clustercmd.zoneid = zoneId - clusterresponse = self.apiClient.addCluster(clustercmd) - if clusterresponse != FAILED and clusterresponse[0].id is not None: + clusterresponse = self.__apiClient.addCluster(clustercmd) + if clusterresponse != FAILED and clusterresponse[0].id: clusterId = clusterresponse[0].id - self.tcRunLogger.\ + self.__tcRunLogger.\ debug("Cluster Name : %s Id : %s Created Successfully" % (str(cluster.clustername), str(clusterId))) + self.__addToCleanUp("Cluster", clusterId) else: - self.tcRunLogger.exception("====Cluster %s Creation Failed" - "=====" % str(cluster.clustername)) - exit(1) + self.__tcRunLogger.exception("====Cluster %s Creation Failed" + "=====" % + str(cluster.clustername)) + self.__cleanAndExit() if cluster.hypervisor.lower() != "vmware": self.addHosts(cluster.hosts, zoneId, podId, clusterId, cluster.hypervisor) @@ -113,16 +181,15 @@ class DeployDataCenters(object): def waitForHost(self, zoneId, clusterId): """ Wait for the hosts in the zoneid, clusterid to be up - 2 retries with 30s delay """ retry, timeout = 2, 30 cmd = listHosts.listHostsCmd() cmd.clusterid, cmd.zoneid = clusterId, zoneId - hosts = self.apiClient.listHosts(cmd) + hosts = self.__apiClient.listHosts(cmd) if hosts == FAILED: - self.tcRunLogger.exception("=== List Hosts Failed===") - exit(1) + self.__tcRunLogger.exception("=== List Hosts Failed===") + self.__cleanAndExit() while retry != 0: for host in hosts: if host.state != 'Up': @@ -146,10 +213,15 @@ class DeployDataCenters(object): primarycmd.url = primary.url primarycmd.zoneid = zoneId primarycmd.clusterid = clusterId - if self.apiClient.createStoragePool(primarycmd) == FAILED: - self.tcRunLogger.\ + ret = self.__apiClient.createStoragePool(primarycmd) + if ret != FAILED and ret.id: + self.__tcRunLogger.debug( + "=== Creating Storage Pool Successful===") + self.__addToCleanUp("StoragePool", ret.id) + else: + self.__tcRunLogger.\ exception("=== Create Storage Pool Failed===") - exit(1) + self.__cleanAndExit() def createPods(self, pods, @@ -165,18 +237,19 @@ class DeployDataCenters(object): createpod.startip = pod.startip createpod.endip = pod.endip createpod.zoneid = zoneId - createpodResponse = self.apiClient.createPod(createpod) + createpodResponse = self.__apiClient.createPod(createpod) if createpodResponse != \ - FAILED and createpodResponse.id is not None: + FAILED and createpodResponse.id: podId = createpodResponse.id - self.tcRunLogger.debug("Pod Name : %s Id : %s " - "Created Successfully" % - (str(pod.name), str(podId))) + self.__tcRunLogger.debug("Pod Name : %s Id : %s " + "Created Successfully" % + (str(pod.name), str(podId))) + self.__addToCleanUp("Pod", podId) else: - self.tcRunLogger.\ + self.__tcRunLogger.\ exception("====Pod: %s Creation " "Failed=====" % str(pod.name)) - exit(1) + self.__cleanAndExit() if pod.guestIpRanges is not None and networkId is not None: self.createVlanIpRanges("Basic", pod.guestIpRanges, zoneId, @@ -208,10 +281,15 @@ class DeployDataCenters(object): vlanipcmd.forvirtualnetwork = "false" else: vlanipcmd.forvirtualnetwork = "true" - if self.apiClient.createVlanIpRange(vlanipcmd) == FAILED: - self.tcRunLogger.\ + ret = self.__apiClient.createVlanIpRange(vlanipcmd) + if ret != FAILED and ret.id: + self.__tcRunLogger.debug( + "=== Creating Vlan Ip Range Successful===") + self.__addToCleanUp("VlanIpRange", ret.id) + else: + self.__tcRunLogger.\ exception("=== Create Vlan Ip Range Failed===") - exit(1) + self.__cleanAndExit() def createSecondaryStorages(self, secondaryStorages, zoneId): if secondaryStorages is None: @@ -231,10 +309,14 @@ class DeployDataCenters(object): }) if secondarycmd.provider == "NFS": secondarycmd.zoneid = zoneId - if self.apiClient.addImageStore(secondarycmd) == FAILED: - self.tcRunLogger.\ + ret = self.__apiClient.addImageStore(secondarycmd) + if ret != FAILED and ret.id: + self.__tcRunLogger.debug("===Add Image Store Successful===") + self.__addToCleanUp("ImageStore", ret.id) + else: + self.__tcRunLogger.\ exception("=== Add Image Store Failed===") - exit(1) + self.__cleanAndExit() def createCacheStorages(self, cacheStorages, zoneId): if cacheStorages is None: @@ -253,11 +335,16 @@ class DeployDataCenters(object): 'key': key, 'value': value }) - if self.apiClient.createSecondaryStagingStore(cachecmd) == FAILED: - self.tcRunLogger.\ - exception("=== Create " + ret = self.__apiClient.createSecondaryStagingStore(cachecmd) + if ret != FAILED and ret.id: + self.__tcRunLogger.debug( + "===Creating Secondary StagingStore Successful===") + self.__addToCleanUp("SecondaryStagingStore", ret.id) + else: + self.__tcRunLogger.\ + exception("=== Creating " "SecondaryStagingStorage Failed===") - exit(1) + self.__cleanAndExit() def createNetworks(self, networks, zoneId): if networks is None: @@ -276,19 +363,19 @@ class DeployDataCenters(object): networkcmd.endip = iprange.endip networkcmd.gateway = iprange.gateway networkcmd.netmask = iprange.netmask - - networkcmdresponse = self.apiClient.createNetwork(networkcmd) + networkcmdresponse = self.__apiClient.createNetwork(networkcmd) if networkcmdresponse != \ - FAILED and networkcmdresponse.id is not None: + FAILED and networkcmdresponse.id: networkId = networkcmdresponse.id - self.tcRunLogger.\ - debug("Network Name : %s Id : %s Created Successfully" + self.__tcRunLogger.\ + debug("Creating Network Name : %s Id : %s Successful" % (str(network.name), str(networkId))) + self.__addToCleanUp("Network", networkId) else: - self.tcRunLogger.\ + self.__tcRunLogger.\ exception("====Network : %s " "Creation Failed=====" % str(network.name)) - exit(1) + self.__cleanAndExit() return networkId def createPhysicalNetwork(self, net, zoneid): @@ -296,16 +383,17 @@ class DeployDataCenters(object): phynet.zoneid = zoneid phynet.name = net.name phynet.isolationmethods = net.isolationmethods - phynetwrk = self.apiClient.createPhysicalNetwork(phynet) - if phynetwrk != FAILED and phynetwrk.id is not None: - self.tcRunLogger.debug("Physical Network Name : %s Id : %s " - "Created Successfully" % - (str(phynet.name), - str(phynetwrk.id))) + phynetwrk = self.__apiClient.createPhysicalNetwork(phynet) + if phynetwrk != FAILED and phynetwrk.id: + self.__tcRunLogger.\ + debug("Creating Physical Network Name : " + "%s Id : %s Successful" % (str(phynet.name), + str(phynetwrk.id))) + self.__addToCleanUp("PhysicalNetwork", phynetwrk.id) else: - self.tcRunLogger.exception("====Physical Network " - "Creation Failed=====") - exit(1) + self.__tcRunLogger.exception("====Physical Network " + "Creation Failed=====") + self.__cleanAndExit() self.addTrafficTypes(phynetwrk.id, net.traffictypes) return phynetwrk @@ -316,23 +404,26 @@ class DeployDataCenters(object): upnet.state = state if vlan: upnet.vlan = vlan - ret = self.apiClient.updatePhysicalNetwork(upnet) + ret = self.__apiClient.updatePhysicalNetwork(upnet) if ret == FAILED: - self.tcRunLogger.\ + self.__tcRunLogger.\ exception("====Update Physical Network Failed=====") - exit(1) - else: - return ret + self.__cleanAndExit() + return ret def enableProvider(self, provider_id): upnetprov =\ updateNetworkServiceProvider.updateNetworkServiceProviderCmd() upnetprov.id = provider_id upnetprov.state = "Enabled" - if self.apiClient.updateNetworkServiceProvider(upnetprov) == FAILED: - self.tcRunLogger.\ + ret = self.__apiClient.updateNetworkServiceProvider(upnetprov) + if ret == FAILED: + self.__tcRunLogger.\ exception("====Update Network Service Provider Failed=====") - exit(1) + self.__cleanAndExit() + else: + self.__tcRunLogger.debug( + "===Update Network Service Provider Successfull===") def configureProviders(self, phynetwrk, providers): """ @@ -345,12 +436,13 @@ class DeployDataCenters(object): pnetprov.physicalnetworkid = phynetwrk.id pnetprov.state = "Disabled" pnetprov.name = provider.name - pnetprovres = self.apiClient.listNetworkServiceProviders(pnetprov) + pnetprovres = self.__apiClient.listNetworkServiceProviders( + pnetprov) if pnetprovres == FAILED: - self.tcRunLogger.\ + self.__tcRunLogger.\ exception("====List Network " "Service Providers Failed=====") - exit(1) + self.__cleanAndExit() if pnetprovres and len(pnetprovres) > 0: if provider.name == 'VirtualRouter'\ @@ -358,7 +450,7 @@ class DeployDataCenters(object): vrprov = listVirtualRouterElements.\ listVirtualRouterElementsCmd() vrprov.nspid = pnetprovres[0].id - vrprovresponse = self.apiClient.\ + vrprovresponse = self.__apiClient.\ listVirtualRouterElements(vrprov) vrprovid = vrprovresponse[0].id @@ -367,34 +459,38 @@ class DeployDataCenters(object): configureVirtualRouterElementCmd() vrconfig.enabled = "true" vrconfig.id = vrprovid - if self.apiClient.\ + if self.__apiClient.\ configureVirtualRouterElement(vrconfig) == \ FAILED: - self.tcRunLogger.\ + self.__tcRunLogger.\ exception("====ConfigureVirtualRouterElement " "Failed=====") - exit(1) + self.__cleanAndExit() self.enableProvider(pnetprovres[0].id) elif provider.name == 'InternalLbVm': internallbprov = listInternalLoadBalancerElements.\ listInternalLoadBalancerElementsCmd() internallbprov.nspid = pnetprovres[0].id - internallbresponse = self.apiClient.\ + internallbresponse = self.__apiClient.\ listInternalLoadBalancerElements(internallbprov) if internallbresponse == FAILED: - self.tcRunLogger.\ + self.__tcRunLogger.\ exception("====List " "InternalLoadBalancerElements " "Failed=====") - exit(1) + self.__cleanAndExit() internallbid = internallbresponse[0].id internallbconfig = \ configureInternalLoadBalancerElement.\ configureInternalLoadBalancerElementCmd() internallbconfig.enabled = "true" internallbconfig.id = internallbid - self.apiClient.\ - configureInternalLoadBalancerElement(internallbconfig) + if self.__apiClient.\ + configureInternalLoadBalancerElement( + internallbconfig) == FAILED: + self.__tcRunLogger.exception( + "==== Configure Internal LB Failed===") + self.__cleanAndExit() self.enableProvider(pnetprovres[0].id) elif provider.name == 'SecurityGroupProvider': self.enableProvider(pnetprovres[0].id) @@ -403,7 +499,18 @@ class DeployDataCenters(object): addNetworkServiceProviderCmd() netprov.name = provider.name netprov.physicalnetworkid = phynetwrk.id - result = self.apiClient.addNetworkServiceProvider(netprov) + result = self.__apiClient.addNetworkServiceProvider(netprov) + if result != FAILED and result.id: + self.__tcRunLogger.\ + debug("==== AddNetworkServiceProvider " + "Successful=====") + self.__addToCleanUp("NetworkServiceProvider", result.id) + else: + self.__tcRunLogger.\ + exception("==== AddNetworkServiceProvider " + "Failed=====") + self.__cleanAndExit() + for device in provider.devices: if provider.name == 'Netscaler': dev = addNetscalerLoadBalancer.\ @@ -413,7 +520,19 @@ class DeployDataCenters(object): dev.networkdevicetype = device.networkdevicetype dev.url = configGenerator.getDeviceUrl(device) dev.physicalnetworkid = phynetwrk.id - self.apiClient.addNetscalerLoadBalancer(dev) + ret = self.__apiClient.addNetscalerLoadBalancer(dev) + if ret != FAILED and ret.id: + self.__tcRunLogger.\ + debug("==== AddNetScalerLB " + "Successful=====") + self.__addToCleanUp( + "NetscalerLoadBalancer", + ret.id) + else: + self.__tcRunLogger.exception( + "===AddNetScalerLB Failed===") + self.__cleanAndExit() + elif provider.name == 'JuniperSRX': dev = addSrxFirewall.addSrxFirewallCmd() dev.username = device.username @@ -421,7 +540,16 @@ class DeployDataCenters(object): dev.networkdevicetype = device.networkdevicetype dev.url = configGenerator.getDeviceUrl(device) dev.physicalnetworkid = phynetwrk.id - self.apiClient.addSrxFirewall(dev) + ret = self.__apiClient.addSrxFirewall(dev) + if ret != FAILED and ret.id: + self.__tcRunLogger.\ + debug("==== AddSrx " + "Successful=====") + self.__addToCleanUp("SrxFirewall", ret.id) + else: + self.__tcRunLogger.exception("===AddSrx Failed===") + self.__cleanAndExit() + elif provider.name == 'F5BigIp': dev = addF5LoadBalancer.addF5LoadBalancerCmd() dev.username = device.username @@ -429,7 +557,15 @@ class DeployDataCenters(object): dev.networkdevicetype = device.networkdevicetype dev.url = configGenerator.getDeviceUrl(device) dev.physicalnetworkid = phynetwrk.id - self.apiClient.addF5LoadBalancer(dev) + ret = self.__apiClient.addF5LoadBalancer(dev) + if ret != FAILED and ret.id: + self.__tcRunLogger.\ + debug("==== AddF5 " + "Successful=====") + self.__addToCleanUp("F5LoadBalancer", ret.id) + else: + self.__tcRunLogger.exception("===AddF5 Failed===") + self.__cleanAndExit() else: raise InvalidParameterException("Device %s " "doesn't match " @@ -453,52 +589,59 @@ class DeployDataCenters(object): if traffictype.vmware is not None else None traffic_type.simulatorlabel = traffictype.simulator\ if traffictype.simulator is not None else None - ret = self.apiClient.addTrafficType(traffic_type) + ret = self.__apiClient.addTrafficType(traffic_type) if ret == FAILED: - self.tcRunLogger.\ + self.__tcRunLogger.\ exception("==== Add TrafficType Failed=====") + self.__cleanAndExit() else: + self.__tcRunLogger.debug("===Add TrafficType Successful====") + self.__addToCleanUp("TrafficType", ret.id) return ret def enableZone(self, zoneid, allocation_state="Enabled"): zoneCmd = updateZone.updateZoneCmd() zoneCmd.id = zoneid zoneCmd.allocationstate = allocation_state - ret = self.apiClient.updateZone(zoneCmd) + ret = self.__apiClient.updateZone(zoneCmd) if ret == FAILED: - self.tcRunLogger.exception("==== Update Zone Failed=====") - else: - return ret + self.__tcRunLogger.exception("==== Update Zone Failed=====") + self.__cleanAndExit() + return ret def updateZoneDetails(self, zoneid, details): zoneCmd = updateZone.updateZoneCmd() zoneCmd.id = zoneid zoneCmd.details = details - ret = self.apiClient.updateZone(zoneCmd) + ret = self.__apiClient.updateZone(zoneCmd) if ret == FAILED: - self.tcRunLogger.exception("==== Update Zone Failed=====") - else: - return ret + self.__tcRunLogger.exception("==== Update Zone Failed=====") + self.__cleanAndExit() + return ret def createZone(self, zone, rec=0): try: - zoneresponse = self.apiClient.createZone(zone) + zoneresponse = self.__apiClient.createZone(zone) if zoneresponse == FAILED: - self.tcRunLogger.\ + self.__tcRunLogger.\ exception("====Zone : %s Creation Failed=====" % str(zone.name)) + print "\n====Zone : %s Creation Failed=====" % str(zone.name) if not rec: - zone.name = zone.name + random_gen() - self.tcRunLogger.\ + zone.name = zone.name + "_" + random_gen() + self.__tcRunLogger.\ debug("====Recreating Zone With New Name : " "%s" % zone.name) + print "\n====Recreating Zone With New Name ====", \ + str(zone.name) return self.createZone(zone, 1) else: - self.tcRunLogger.\ + self.__addToCleanUp("Zone", zoneresponse.id) + self.__tcRunLogger.\ debug("Zone Name : %s Id : %s Created Successfully" % (str(zone.name), str(zoneresponse.id))) return zoneresponse.id - except Exception, e: + except Exception as e: print "\nException Occurred under createZone %s" % \ GetDetailExceptionInfo(e) return FAILED @@ -516,18 +659,18 @@ class DeployDataCenters(object): zonecmd.networktype = zone.networktype if zone.securitygroupenabled != "true": zonecmd.guestcidraddress = zone.guestcidraddress - ret = self.createZone(zonecmd) - if ret == FAILED: - self.tcRunLogger.\ - exception("====Zone Creation Failed. So Exiting=====") - exit(1) - zoneId = ret + zoneId = self.createZone(zonecmd) + if zoneId == FAILED: + self.__tcRunLogger.\ + exception( + "====Zone: %s Creation Failed. So Exiting=====" % + str(zone.name)) + self.__cleanAndExit() for pnet in zone.physical_networks: phynetwrk = self.createPhysicalNetwork(pnet, zoneId) self.configureProviders(phynetwrk, pnet.providers) self.updatePhysicalNetwork(phynetwrk.id, "Enabled", vlan=pnet.vlan) - if zone.networktype == "Basic": listnetworkoffering =\ listNetworkOfferings.listNetworkOfferingsCmd() @@ -542,12 +685,12 @@ class DeployDataCenters(object): listnetworkoffering.name = zone.networkofferingname listnetworkofferingresponse = \ - self.apiClient.listNetworkOfferings(listnetworkoffering) + self.__apiClient.listNetworkOfferings(listnetworkoffering) if listnetworkofferingresponse == FAILED: - self.tcRunLogger.\ + self.__tcRunLogger.\ exception("==== " "ListNetworkOfferingResponse Failed=====") - exit(1) + self.__cleanAndExit() guestntwrk = configGenerator.network() guestntwrk.displaytext = "guestNetworkForBasicZone" guestntwrk.name = "guestNetworkForBasicZone" @@ -575,14 +718,13 @@ class DeployDataCenters(object): "DefaultSharedNetworkOfferingWithSGService" if zone.networkofferingname is not None: listnetworkoffering.name = zone.networkofferingname - listnetworkofferingresponse = \ - self.apiClient.listNetworkOfferings(listnetworkoffering) + self.__apiClient.listNetworkOfferings(listnetworkoffering) if listnetworkofferingresponse == FAILED: - self.tcRunLogger.\ + self.__tcRunLogger.\ exception("==== ListNetworkOfferingResponse " "Failed=====") - exit(1) + self.__cleanAndExit() networkcmd = createNetwork.createNetworkCmd() networkcmd.displaytext = "Shared SG enabled network" networkcmd.name = "Shared SG enabled network" @@ -599,18 +741,18 @@ class DeployDataCenters(object): networkcmd.netmask = iprange.netmask networkcmd.vlan = iprange.vlan - networkcmdresponse = self.apiClient.createNetwork(networkcmd) + networkcmdresponse = self.__apiClient.createNetwork(networkcmd) if networkcmdresponse != \ - FAILED and networkcmdresponse.id is not None: - networkId = networkcmdresponse.id - self.tcRunLogger.\ - debug("Network Id : %s " - "Created Successfully" % str(networkId)) + FAILED and networkcmdresponse.id: + self.__addToCleanUp("Network", networkcmdresponse.id) + self.__tcRunLogger.\ + debug("create Network Successful. NetworkId : %s " + % str(networkcmdresponse.id)) else: - self.tcRunLogger.\ - exception("====Network Creation Failed=====") - exit(1) - self.createPods(zone.pods, zoneId, networkId) + self.__tcRunLogger.\ + exception("====Create Network Failed=====") + self.__cleanAndExit() + self.createPods(zone.pods, zoneId, networkcmdresponse.id) '''Note: Swift needs cache storage first''' self.createCacheStorages(zone.cacheStorages, zoneId) @@ -623,7 +765,6 @@ class DeployDataCenters(object): if details is not None: det = [d.__dict__ for d in details] self.updateZoneDetails(zoneId, det) - return def isEipElbZone(self, zone): @@ -634,20 +775,27 @@ class DeployDataCenters(object): return False def setClient(self): - self.apiClient = self.testClient.getApiClient() + ''' + @Name : setClient + @Desc : Sets the API Client retrieved from test client + ''' + self.__apiClient = self.__testClient.getApiClient() def updateConfiguration(self, globalCfg): - if globalCfg is None or self.apiClient is None: + if globalCfg is None or self.__apiClient is None: return None for config in globalCfg: updateCfg = updateConfiguration.updateConfigurationCmd() updateCfg.name = config.name updateCfg.value = config.value - if self.apiClient.updateConfiguration(updateCfg) == FAILED: - self.tcRunLogger.\ + if self.__apiClient.updateConfiguration(updateCfg) == FAILED: + self.__tcRunLogger.\ exception("===UpdateConfiguration Failed===") - exit(1) + self.__cleanAndExit() + else: + self.__tcRunLogger.debug( + "==UpdateConfiguration Successfull===") def copyAttributesToCommand(self, source, command): map(lambda attr: setattr(command, attr, getattr(source, attr, None)), @@ -659,22 +807,191 @@ class DeployDataCenters(object): return command = addS3.addS3Cmd() self.copyAttributesToCommand(s3, command) - if self.apiClient.addS3(command) == FAILED: - self.tcRunLogger.exception("====AddS3 Failed===") - exit(1) + ret = self.__apiClient.addS3(command) + if ret != FAILED and ret.id: + self.__tcRunLogger.debug("===AddS3 Successfull===") + self.__addToCleanUp("s3", ret.id) + else: + self.__tcRunLogger.exception("====AddS3 Failed===") + self.__cleanAndExit() def deploy(self): try: + print "\n==== Deploy DC Started ====" + self.__tcRunLogger.debug("\n==== Deploy DC Started ====") self.setClient() - self.updateConfiguration(self.config.globalConfig) - self.createZones(self.config.zones) - self.configureS3(self.config.s3) + self.updateConfiguration(self.__config.globalConfig) + self.createZones(self.__config.zones) + self.configureS3(self.__config.s3) + self.__persistDcConfig() + self.__tcRunLogger.debug("\n====Deploy DC Successful====") + print "\n====Deploy DC Successful=====" return SUCCESS - except Exception, e: + except Exception as e: print "\nException Occurred Under deploy :%s" % \ GetDetailExceptionInfo(e) + self.__tcRunLogger.debug("\n====Deploy DC Failed====") + print "\n====Deploy DC Failed====" + self.__cleanAndExit() return FAILED + +class DeleteDataCenter: + + ''' + @Desc : Deletes the Data Center using the settings provided. + Reads the Configuration provided from file + Uses the read configuration to delete the DataCenter + ''' + + def __init__(self, + tc_client, + dc_cfg_file=None, + dc_cfg=None, + tc_run_logger=None + ): + self.__dcCfgFile = dc_cfg_file + self.__dcCfg = dc_cfg + self.__tcRunLogger = tc_run_logger + self.__apiClient = None + self.__testClient = tc_client + + def __deleteCmds(self, cmd_name, cmd_obj): + ''' + @Name : __deleteCmds + @Desc : Deletes the entities provided by cmd + ''' + if cmd_name == "deleteHostCmd": + cmd_obj.forcedestroylocalstorage = "true" + cmd_obj.force = "true" + ''' + Step1 : Prepare Host For Maintenance + ''' + host_maint_cmd = prepareHostForMaintenance.\ + prepareHostForMaintenanceCmd() + host_maint_cmd.id = cmd_obj.id + host_maint_resp = self.__apiClient.prepareHostForMaintenance( + host_maint_cmd) + if host_maint_resp != FAILED: + ''' + Step2 : List Hosts for Resource State + ''' + list_host_cmd = listHosts.listHostsCmd() + list_host_cmd.id = cmd_obj.id + retries = 3 + for i in xrange(retries): + list_host_resp = self.__apiClient.\ + listHosts(list_host_cmd) + if (list_host_resp != FAILED) and\ + (list_host_resp[0].resourcestate == 'Maintenance'): + break + sleep(30) + if cmd_name == "deleteStorage": + cmd_obj.forced = "true" + store_maint_cmd = enableStorageMaintenance.\ + enableStorageMaintenanceCmd() + store_maint_cmd.id = cmd_obj.id + store_maint_resp = self.__apiClient.\ + enableStorageMaintenance(store_maint_cmd) + if store_maint_resp != FAILED: + list_store_cmd = listStoragePools.listStoragePoolsCmd() + list_store_cmd.id = cmd_obj.id + retries = 3 + for i in xrange(retries): + store_maint_resp = self.__apiClient.\ + listStoragePools(list_store_cmd) + if (store_maint_resp != FAILED) and \ + (store_maint_resp[0].state == 'Maintenance'): + break + sleep(30) + return cmd_obj + + def __setClient(self): + ''' + @Name : setClient + @Desc : Sets the API Client retrieved from test client + ''' + self.__apiClient = self.__testClient.getApiClient() + + def __cleanEntries(self): + ''' + @Name : __cleanAndEntries + @Description: Cleans up the created DC in order of creation + ''' + try: + ret = FAILED + if "order" in self.__dcCfg.keys() and len(self.__dcCfg["order"]): + self.__dcCfg["order"].reverse() + print "\n====Clean Up Entries===", self.__dcCfg + for type in self.__dcCfg["order"]: + self.__tcRunLogger.debug( + "====CleanUp Started For Type: %s====" % + type) + if type: + temp_ids = self.__dcCfg[type] + ids = [items for items in temp_ids if items] + for id in ids: + del_mod = "delete" + type + del_cmd = getattr( + globals()[del_mod], + del_mod + "Cmd") + del_cmd_obj = del_cmd() + del_cmd_resp = getattr( + globals()[del_mod], + del_mod + "Response") + del_cmd_obj.id = id + del_cmd_obj = self.__deleteCmds( + del_mod + + "Cmd", + del_cmd_obj) + del_func = getattr(self.__apiClient, del_mod) + del_cmd_resp = del_func(del_cmd_obj) + if del_cmd_resp == FAILED: + self.__tcRunLogger.debug( + "====%s CleanUp Failed. ID: %s ===" % + (type, id)) + else: + self.__tcRunLogger.debug( + "====%s CleanUp Successful. ID : %s===" % + (type, id)) + ret = SUCCESS + except Exception as e: + print "\n==== Exception Under __cleanEntries: %s ==== % " \ + % GetDetailExceptionInfo(e) + self.__tcRunLogger.exception( + "\n==== Exception Under __cleanEntries: %s ==== % " % + GetDetailExceptionInfo(e)) + finally: + return ret + + def removeDataCenter(self): + ''' + @Name : removeDataCenter + @Desc : Removes the Data Center provided by Configuration + If Input dc file configuration is None, uses the cfg provided + else uses the dc file to get the configuration + ''' + try: + self.__setClient() + self.__tcRunLogger.debug("====DeployDC: CleanUp Started====") + print "\n====DeployDC: CleanUp Started====" + ret = FAILED + if self.__dcCfgFile: + file_to_read = open(self.__dcCfgFile, 'r') + if file_to_read: + self.__dcCfg = pickle.load(file_to_read) + if self.__dcCfg: + ret = self.__cleanEntries() + except Exception as e: + print "\n==== Exception Under removeDataCenter: %s ====" % \ + GetDetailExceptionInfo(e) + self.__tcRunLogger.exception( + "===DeployDC CleanUp FAILED: %s ====" % + GetDetailExceptionInfo(e)) + finally: + return ret + + if __name__ == "__main__": ''' @Desc : This facility is mainly to deploy @@ -683,62 +1000,83 @@ if __name__ == "__main__": ''' parser = OptionParser() parser.add_option("-i", "--input", action="store", - default="./datacenterCfg", dest="input", + default=None, dest="input", help="the path \ - where the json config file generated, by default is \ - ./datacenterCfg") + where the json config file generated") + parser.add_option("-r", "--remove", action="store", + default=None, dest="remove", + help="path to file\ + where the created dc entries are kept") (options, args) = parser.parse_args() - if options.input: - ''' - Imports the Modules Required - ''' - from marvin.marvinLog import MarvinLog - from marvin.cloudstackTestClient import CSTestClient - ''' - Step1 : Parse and Create the config from input config provided - ''' - cfg = configGenerator.getSetupConfig(options.input) - log_obj = MarvinLog("CSLog") - log_check = False - if log_obj is not None: - log_check = True - ret = log_obj.createLogs("DeployDataCenter", - cfg.logger) - if ret != FAILED: - log_folder_path = log_obj.getLogFolderPath() - tc_run_logger = log_obj.getLogger() - if log_check is False: - print "\nLog Creation Failed. Please Check" - else: - print "\nAll Logs will be available under %s" % \ - str(log_folder_path) - ''' - Step2: Create the Test Client - ''' - obj_tc_client = CSTestClient(cfg.mgtSvr[0], cfg.dbSvr, - logger=tc_run_logger) - if obj_tc_client is not None and obj_tc_client.createTestClient() \ - != FAILED: - deploy = DeployDataCenters(obj_tc_client, cfg, tc_run_logger) - if deploy.deploy() == FAILED: - print "\nDeploy DC Failed" - exit(1) - else: - print "\nTestClient Creation Failed. Please Check" - exit(1) + + ''' + Verify the input validity + ''' + if options.input is None and options.remove is None: + print "\n==== For DeployDataCenter: Please Specify a " \ + "Valid Input Configuration File====" + print "\n==== For DeleteDataCenter: Please Specify a " \ + "Valid DC Exported Configuration Settings or File====" + exit(1) + + ''' + Imports the Modules Required + ''' + from marvin.marvinLog import MarvinLog + from marvin.cloudstackTestClient import CSTestClient + + ''' + Step1: Create the Logger + ''' + log_obj = MarvinLog("CSLog") + cfg = configGenerator.getSetupConfig(options.input) + log = cfg.logger + + ret = log_obj.createLogs("DeployDataCenter", + log) + if ret != FAILED: + log_folder_path = log_obj.getLogFolderPath() + tc_run_logger = log_obj.getLogger() else: - print "\n Please Specify a Valid Input Configuration File" + print "\nLog Creation Failed. Please Check" + exit(1) - """ - create = createStoragePool.createStoragePoolCmd() - create.clusterid = 1 - create.podid = 2 - create.name = "fdffdf" - create.url = "nfs://jfkdjf/fdkjfkd" - create.zoneid = 2 + obj_tc_client = CSTestClient(cfg.mgtSvr[0], cfg.dbSvr, + logger=tc_run_logger) + if obj_tc_client and obj_tc_client.createTestClient() == FAILED: + print "\n=== TestClient Creation Failed===" + exit(1) - deploy = deployDataCenters("./datacenterCfg") - deploy.loadCfg() - deploy.apiClient.createStoragePool(create) - """ + ''' + Step3: Verify and continue whether to deploy a DC or remove a DC + ''' + + if (options.input) and (os.path.isfile(options.input)) and \ + (options.remove is None): + ''' + @Desc : Deploys a Data Center with provided Config + ''' + deploy = DeployDataCenters(obj_tc_client, + cfg, + tc_run_logger, + log_folder_path=log_folder_path) + deploy.deploy() + exit(1) + + if options.remove and os.path.isfile(options.remove) and options.input: + ''' + @Desc : Removes a Data Center with provided Config + ''' + remove_dc_obj = DeleteDataCenter(obj_tc_client, + dc_cfg_file=options.remove, + tc_run_logger=tc_run_logger + ) + if remove_dc_obj: + if remove_dc_obj.removeDataCenter() == FAILED: + print "\n===Removing DataCenter Failed===" + tc_run_logger.debug("\n===Removing DataCenter Failed===") + else: + print "\n===Removing DataCenter Successful===" + tc_run_logger.debug("\n===Removing DataCenter Successful===") + exit(1) diff --git a/tools/marvin/marvin/marvinLog.py b/tools/marvin/marvin/marvinLog.py index 80120c9b6ab..f62c35a9593 100644 --- a/tools/marvin/marvin/marvinLog.py +++ b/tools/marvin/marvin/marvinLog.py @@ -143,13 +143,12 @@ class MarvinLog: else: temp_path = str(test_module_name) + "__" + str(temp_ts) + temp_dir = "MarvinLogs" if ((log_cfg is not None) and ('LogFolderPath' in log_cfg.__dict__.keys()) and (log_cfg.__dict__.get('LogFolderPath') is not None)): temp_dir = \ log_cfg.__dict__.get('LogFolderPath') + "/MarvinLogs" - else: - temp_dir = "MarvinLogs" self.__logFolderDir = temp_dir + "//" + temp_path print "\n*********Log Folder Path: %s. " \ diff --git a/tools/marvin/marvin/marvinPlugin.py b/tools/marvin/marvin/marvinPlugin.py index e6986cc0277..1ba883a28bb 100644 --- a/tools/marvin/marvin/marvinPlugin.py +++ b/tools/marvin/marvin/marvinPlugin.py @@ -30,6 +30,7 @@ from marvin.cloudstackException import GetDetailExceptionInfo class MarvinPlugin(Plugin): + """ Custom plugin for the cloudstackTestCases to be run using nose """ @@ -53,7 +54,7 @@ class MarvinPlugin(Plugin): ''' self.__deployDcFlag = None self.conf = None - self.__debugStream = stdout + self.__resultStream = stdout self.__testRunner = None self.__testResult = SUCCESS self.__startTime = None @@ -128,7 +129,7 @@ class MarvinPlugin(Plugin): os.system("python " + filename) return True return False - except ImportError, e: + except ImportError as e: print "FileName :%s : Error : %s" % \ (filename, GetDetailExceptionInfo(e)) return False @@ -161,15 +162,28 @@ class MarvinPlugin(Plugin): str(self.__testName) + " :::::::::::") self.__startTime = time.time() + def printMsg(self, status, tname, err): + if self.__tcRunLogger: + self.__tcRunLogger.\ + fatal("%s: %s: %s" % (status, + tname, + GetDetailExceptionInfo(err))) + write_str = "=== TestName: %s | Status : %s ===\n" % (tname, status) + self.__resultStream.write(write_str) + print write_str + + def addSuccess(test): + ''' + Adds the Success Messages to logs + ''' + printMsg(SUCCESS, self.__testName, "Test Case Passed") + self.__testresult = SUCCESS + def handleError(self, test, err): ''' Adds Exception throwing test cases and information to log. ''' - if self.__tcRunLogger: - self.__tcRunLogger.\ - fatal("%s: %s: %s" % (EXCEPTION, - self.__testName, - GetDetailExceptionInfo(err))) + self.printMsg(EXCEPTION, self.__testName, GetDetailExceptionInfo(err)) self.__testResult = EXCEPTION def prepareTestRunner(self, runner): @@ -180,10 +194,7 @@ class MarvinPlugin(Plugin): ''' Adds Failing test cases and information to log. ''' - if self.__tcRunLogger: - self.__tcRunLogger.\ - fatal("%s: %s: %s" % - (FAILED, self.__testName, GetDetailExceptionInfo(err))) + self.printMsg(FAILED, self.__testName, GetDetailExceptionInfo(err)) self.__testResult = FAILED def __getModName(self, inp, type='file'): @@ -202,7 +213,7 @@ class MarvinPlugin(Plugin): def __runSuite(self, test_suite=None): try: if test_suite: - if self.wantFile(test_suite) is True: + if self.wantFile(test_suite): test_mod_name = self.__getModName(test_suite) temp_obj = MarvinInit(self.__configFile, None, test_mod_name, @@ -213,14 +224,14 @@ class MarvinPlugin(Plugin): self.__testClient = temp_obj.getTestClient() self.__tcRunLogger = temp_obj.getLogger() self.__parsedConfig = temp_obj.getParsedConfig() - self.__debugStream = temp_obj.getResultFile() + self.__resultStream = temp_obj.getResultFile() self.__testRunner = nose.core.\ - TextTestRunner(stream=self.__debugStream, + TextTestRunner(stream=self.__resultStream, descriptions=True, verbosity=2) return SUCCESS return FAILED - except Exception, e: + except Exception as e: print "\n Exception Occurred when running suite :%s Error : %s" \ % (test_suite, GetDetailExceptionInfo(e)) return FAILED @@ -259,7 +270,7 @@ class MarvinPlugin(Plugin): if os.path.isfile(suites): self.__runSuite(suites) return SUCCESS - except Exception, e: + except Exception as e: print "Exception Occurred under startMarvin: %s" % \ GetDetailExceptionInfo(e) return FAILED