diff --git a/tools/marvin/marvin/cloudstackTestClient.py b/tools/marvin/marvin/cloudstackTestClient.py index 36f7f8d8369..d9fa2061f65 100644 --- a/tools/marvin/marvin/cloudstackTestClient.py +++ b/tools/marvin/marvin/cloudstackTestClient.py @@ -22,6 +22,18 @@ from cloudstackAPI import * import random import string import hashlib +from configGenerator import ConfigManager + +''' +@Desc : CloudStackTestClient is encapsulated class for getting various \ + clients viz., apiclient,dbconnection etc +@Input : mgmtDetails : Management Server Details + dbSvrDetails: Database Server details of Management \ + Server. Retrieved from configuration file. + asyncTimeout : + defaultWorkerThreads : + logging : +''' class cloudstackTestClient(object): @@ -35,6 +47,21 @@ class cloudstackTestClient(object): self.apiClient =\ cloudstackAPIClient.CloudStackAPIClient(self.connection) self.dbConnection = None + if dbSvrDetails is not None: + self.createDbConnection(dbSvrDetails.dbSvr, dbSvrDetails.port, + dbSvrDetails.user, + dbSvrDetails.passwd, dbSvrDetails.db) + ''' + Provides the Configuration Object to users through getConfigParser + The purpose of this object is to parse the config + and provide dictionary of the config so users can + use that configuration.Users can later call getConfig + on this object and it will return the default parsed + config dictionary from default configuration file, + they can overwrite it with providing their own + configuration file as well. + ''' + self.configObj = ConfigManager() self.asyncJobMgr = None self.ssh = None self.id = None @@ -48,11 +75,10 @@ class cloudstackTestClient(object): def identifier(self, id): self.id = id - def dbConfigure(self, host="localhost", port=3306, user='cloud', - passwd='cloud', db='cloud'): - self.dbConnection = dbConnection.dbConnection(host, port, user, passwd, - db) - + def createDbConnection(self, host="localhost", port=3306, user='cloud', + passwd='cloud', db='cloud'): + self.dbConnection = dbConnection.dbConnection(host, port, user, + passwd, db) def isAdminContext(self): """ A user is a regular user if he fails to listDomains; @@ -149,16 +175,8 @@ class cloudstackTestClient(object): def getDbConnection(self): return self.dbConnection - def executeSql(self, sql=None): - if sql is None or self.dbConnection is None: - return None - - return self.dbConnection.execute() - - def executeSqlFromFile(self, sqlFile=None): - if sqlFile is None or self.dbConnection is None: - return None - return self.dbConnection.executeSqlFromFile(sqlFile) + def getConfigParser(self): + return self.configObj def getApiClient(self): self.apiClient.id = self.identifier diff --git a/tools/marvin/marvin/config/setup.cfg b/tools/marvin/marvin/config/setup.cfg new file mode 100644 index 00000000000..216314ff6bc --- /dev/null +++ b/tools/marvin/marvin/config/setup.cfg @@ -0,0 +1,228 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +{ + "zones": [ + { + "name": "Sandbox-simulator", + "guestcidraddress": "10.1.1.0/24", + "dns1": "10.147.28.6", + "physical_networks": [ + { + "broadcastdomainrange": "Zone", + "vlan": "100-200", + "name": "Sandbox-pnet", + "traffictypes": [ + { + "typ": "Guest" + }, + { + "typ": "Management" + }, + { + "typ": "Public" + } + ], + "providers": [ + { + "broadcastdomainrange": "ZONE", + "name": "VirtualRouter" + }, + { + "broadcastdomainrange": "ZONE", + "name": "VpcVirtualRouter" + }, + { + "broadcastdomainrange": "ZONE", + "name": "InternalLbVm" + } + ], + "isolationmethods": [ + "VLAN" + ] + } + ], + "ipranges": [ + { + "startip": "192.168.2.2", + "endip": "192.168.2.200", + "netmask": "255.255.255.0", + "vlan": "50", + "gateway": "192.168.2.1" + } + ], + "networktype": "Advanced", + "pods": [ + { + "endip": "172.16.15.200", + "name": "POD0", + "startip": "172.16.15.2", + "netmask": "255.255.255.0", + "clusters": [ + { + "clustername": "C0", + "hypervisor": "simulator", + "hosts": [ + { + "username": "root", + "url": "http://sim/c0/h0", + "password": "password" + }, + { + "username": "root", + "url": "http://sim/c0/h1", + "password": "password" + } + ], + "clustertype": "CloudManaged", + "primaryStorages": [ + { + "url": "nfs://10.147.28.6:/export/home/sandbox/primary0", + "name": "PS0" + }, + { + "url": "nfs://10.147.28.6:/export/home/sandbox/primary1", + "name": "PS1" + } + ] + }, + { + "clustername": "C1", + "hypervisor": "simulator", + "hosts": [ + { + "username": "root", + "url": "http://sim/c1/h0", + "password": "password" + } + ], + "clustertype": "CloudManaged", + "primaryStorages": [ + { + "url": "nfs://10.147.28.6:/export/home/sandbox/primary2", + "name": "PS2" + } + ] + } + ], + "gateway": "172.16.15.1" + } + ], + "internaldns1": "10.147.28.6", + "secondaryStorages": [ + { + "url": "nfs://10.147.28.6:/export/home/sandbox/secondary", + "provider" : "NFS" + } + ] + } + ], + "dbSvr": { + "dbSvr": "localhost", + "passwd": "cloud", + "db": "cloud", + "port": 3306, + "user": "cloud" + }, + "logger": [ + { + "name": "TestClient", + "file": "/tmp/testclient.log" + }, + { + "name": "TestCase", + "file": "/tmp/testcase.log" + } + ], + "globalConfig": [ + { + "name": "network.gc.wait", + "value": "60" + }, + { + "name": "storage.cleanup.interval", + "value": "300" + }, + { + "name": "vm.op.wait.interval", + "value": "5" + }, + { + "name": "default.page.size", + "value": "10000" + }, + { + "name": "network.gc.interval", + "value": "60" + }, + { + "name": "instance.name", + "value": "QA" + }, + { + "name": "workers", + "value": "10" + }, + { + "name": "account.cleanup.interval", + "value": "600" + }, + { + "name": "guest.domain.suffix", + "value": "sandbox.simulator" + }, + { + "name": "expunge.delay", + "value": "60" + }, + { + "name": "vm.allocation.algorithm", + "value": "random" + }, + { + "name": "expunge.interval", + "value": "60" + }, + { + "name": "expunge.workers", + "value": "3" + }, + { + "name": "check.pod.cidrs", + "value": "true" + }, + { + "name": "secstorage.allowed.internal.sites", + "value": "10.147.28.0/24" + }, + { + "name": "direct.agent.load.size", + "value": "1000" + } + ], + "mgtSvr": [ + { + "mgtSvrIp": "localhost", + "passwd": "password", + "user": "root", + "port": 8096, + "hypervisor": "simulator", + "useHttps": "False", + "certCAPath": "NA", + "certPath": "NA" + } + ] +} diff --git a/tools/marvin/marvin/configGenerator.py b/tools/marvin/marvin/configGenerator.py index a966ae089e4..590b64f6043 100644 --- a/tools/marvin/marvin/configGenerator.py +++ b/tools/marvin/marvin/configGenerator.py @@ -19,6 +19,7 @@ import json import os from optparse import OptionParser import jsonHelper +from marvin.codes import * class managementServer(object): @@ -79,7 +80,7 @@ class zone(object): self.cacheStorages = [] -class traffictype(object): +class trafficType(object): def __init__(self, typ, labeldict=None): self.typ = typ # Guest/Management/Public if labeldict: @@ -150,7 +151,7 @@ class host(object): self.memory = None -class physical_network(object): +class physicalNetwork(object): def __init__(self): self.name = None self.tags = [] @@ -296,6 +297,126 @@ class bigip(object): for r in req]) +class ConfigManager(object): + + ''' + @Name: configManager + @Desc: 1. It provides the basic configuration facilities to marvin. + 2. User can just add configuration files for his tests, deployment + etc, under one config folder before running their tests. + cs/tools/marvin/marvin/config. + They can remove all hard coded values from code and separate + it out as config at this location. + Either add this to the existing setup.cfg as separate section + or add new configuration. + 3. This will thus removes hard coded tests and separate + data from tests. + 4. This API is provided as an additional facility under + cloudstackTestClient and users can get the + configuration object as similar to apiclient,dbconnection + etc to drive their test. + 5. They just add their configuration for a test, + setup etc,at one single place under configuration dir + and use "getConfigParser" API of cloudstackTestClient + It will give them "configObj".They can either pass their own + config file for parsing to "getConfig" or it will use + default config file @ config/setup.cfg. + 6. They will then get the dictionary of parsed + configuration and can use it further to drive their tests or + config drive + 7. Test features, can drive their setups thus removing hard coded + values. Configuration default file will be under config and as + setup.cfg. + 8. Users can use their own configuration file passed to + "getConfig" API,once configObj is returned. + ''' + + def __init__(self): + self.filePath = "config/config.cfg" + self.parsedDict = None + if self.__verifyFile(self.filePath) is not False: + self.parsedDict = self.__parseConfig(self.filePath) + + def __parseConfig(self, file): + ''' + @Name : __parseConfig + @Description: Parses the Input configuration Json file + and returns a dictionary from the file. + @Input : NA + @Output : Returns the parsed dictionary from json file + Returns None for invalid input or if parsing failed + ''' + config_dict = None + try: + configlines = [] + with open(file, 'r') as fp: + for line in fp: + if len(line) != 0: + ws = line.strip() + if ws[0] not in ["#"]: + configlines.append(ws) + config_dict = json.loads("\n".join(configlines)) + except Exception, e: + #Will replace with log once we have logging done + print "\n Exception occurred under __parseConfig", e + finally: + return config_dict + + def __verifyFile(self, file): + ''' + @Name : __parseConfig + @Description: Parses the Input configuration Json file + and returns a dictionary from the file. + @Input : file NA + @Output : True or False based upon file input validity + and availability + ''' + if file is None or file == '': + return False + if os.path.exists(file) is False: + return False + return True + + def __getSectionData(self, return_dict, section=None): + ''' + @Name: getSectionData + @Desc: Gets the Section data of a particular section + under parsed dictionary + @Input: Parsed Dictionary from configuration file + section to be returned from this dict + @Output:Section matching inside the parsed data + ''' + if return_dict is not None: + inp = return_dict + elif self.parsedDict is None: + return INVALID_INPUT + else: + inp = self.parsedDict + + if section is not None: + return inp.get(section) + else: + return inp + + def getConfig(self, file_path=None, section=None): + ''' + @Name: getConfig + @Desc : Parses and converts the given configuration file to dictionary + @Input : file_path: path where the configuration needs to be passed + section: specific section inside the file + @Output: INVALID_INPUT: This value is returned if the input + is invalid or not able to be parsed + Parsed configuration dictionary from json file + ''' + ret = None + if file not in [None, '']: + if self.__verifyFile(file_path) is False: + return INVALID_INPUT + else: + ret = self.__parseConfig(file_path) + return self.__getSectionData(ret, section) + + def getDeviceUrl(obj): req = zip(obj.__dict__.keys(), obj.__dict__.values()) if obj.hostname: @@ -306,7 +427,7 @@ def getDeviceUrl(obj): return None -def describe_setup_in_basic_mode(): +def descSetupInBasicMode(): '''sample code to generate setup configuration file''' zs = cloudstackConfiguration() @@ -325,9 +446,9 @@ def describe_setup_in_basic_mode(): sgprovider.broadcastdomainrange = 'Pod' sgprovider.name = 'SecurityGroupProvider' - pn = physical_network() + pn = physicalNetwork() pn.name = "test-network" - pn.traffictypes = [traffictype("Guest"), traffictype("Management")] + pn.traffictypes = [trafficType("Guest"), trafficType("Management")] pn.providers.append(sgprovider) z.physical_networks.append(pn) @@ -425,7 +546,7 @@ def describe_setup_in_basic_mode(): return zs -def describe_setup_in_eip_mode(): +def descSetupInEipMode(): """ Setting up an EIP/ELB enabled zone with netscaler provider """ @@ -459,12 +580,12 @@ def describe_setup_in_eip_mode(): ns.hostname = '10.147.40.100' nsprovider.devices.append(ns) - pn = physical_network() + pn = physicalNetwork() pn.name = "test-network" - pn.traffictypes = [traffictype("Guest", + pn.traffictypes = [trafficType("Guest", {"xen": "cloud-guest"}), - traffictype("Management"), - traffictype("Public", {"xen": "cloud-public"})] + trafficType("Management"), + trafficType("Public", {"xen": "cloud-public"})] pn.providers.extend([sgprovider, nsprovider]) z.physical_networks.append(pn) @@ -560,7 +681,7 @@ def describe_setup_in_eip_mode(): return zs -def describe_setup_in_advanced_mode(): +def descSetupInAdvancedMode(): '''sample code to generate setup configuration file''' zs = cloudstackConfiguration() @@ -575,10 +696,10 @@ def describe_setup_in_advanced_mode(): z.guestcidraddress = "10.1.1.0/24" z.vlan = "100-2000" - pn = physical_network() + pn = physicalNetwork() pn.name = "test-network" - pn.traffictypes = [traffictype("Guest"), traffictype("Management"), - traffictype("Public")] + pn.traffictypes = [trafficType("Guest"), trafficType("Management"), + trafficType("Public")] vpcprovider = provider('VpcVirtualRouter') @@ -694,7 +815,7 @@ def describe_setup_in_advanced_mode(): '''sample code to generate setup configuration file''' -def describe_setup_in_advancedsg_mode(): +def descSetupInAdvancedsgMode(): zs = cloudstackConfiguration() for l in range(1): @@ -708,9 +829,9 @@ def describe_setup_in_advancedsg_mode(): z.vlan = "100-2000" z.securitygroupenabled = "true" - pn = physical_network() + pn = physicalNetwork() pn.name = "test-network" - pn.traffictypes = [traffictype("Guest"), traffictype("Management")] + pn.traffictypes = [trafficType("Guest"), trafficType("Management")] #If security groups are reqd sgprovider = provider() @@ -861,10 +982,10 @@ by default is ./datacenterCfg") if options.inputfile: config = get_setup_config(options.inputfile) if options.advanced: - config = describe_setup_in_advanced_mode() + config = descSetupInAdvancedMode() elif options.advancedsg: - config = describe_setup_in_advancedsg_mode() + config = descSetupInAdvancedsgMode() else: - config = describe_setup_in_basic_mode() + config = descSetupInBasicMode() generate_setup_config(config, options.output) diff --git a/tools/marvin/marvin/integration/lib/utils.py b/tools/marvin/marvin/integration/lib/utils.py index d81e80d8025..8a1ebbd0c8d 100644 --- a/tools/marvin/marvin/integration/lib/utils.py +++ b/tools/marvin/marvin/integration/lib/utils.py @@ -318,3 +318,36 @@ def is_snapshot_on_nfs(apiclient, dbconn, config, zoneid, snapshotid): raise Exception("SSH failed for management server: %s - %s" % (config.mgtSvr[0].mgtSvrIp, e)) return 'snapshot exists' in result + +def validateList(inp): + ''' + @name: validateList + @Description: 1. A utility function to validate + whether the input passed is a list + 2. The list is empty or not + 3. If it is list and not empty, return PASS and first element + 4. If not reason for FAIL + @Input: Input to be validated + @output: List, containing [ Result,FirstElement,Reason ] + Ist Argument('Result') : FAIL : If it is not a list + If it is list but empty + PASS : If it is list and not empty + IInd Argument('FirstElement'): If it is list and not empty, + then first element + in it, default to None + IIIrd Argument( 'Reason' ): Reason for failure ( FAIL ), + default to None. + INVALID_INPUT + EMPTY_LIST + ''' + ret = [FAIL, None, None] + if inp is None: + ret[2] = INVALID_INPUT + return ret + if not isinstance(inp, list): + ret[2] = INVALID_INPUT + return ret + if len(inp) == 0: + ret[2] = EMPTY_LIST + return ret + return [PASS, inp[0], None]