diff --git a/tools/marvin/marvin/cloudstackTestClient.py b/tools/marvin/marvin/cloudstackTestClient.py index be93f3581cc..3e833c7ecfd 100644 --- a/tools/marvin/marvin/cloudstackTestClient.py +++ b/tools/marvin/marvin/cloudstackTestClient.py @@ -19,6 +19,10 @@ import cloudstackConnection import asyncJobMgr import dbConnection from cloudstackAPI import * +import random +import string +import hashlib +from configGenerator import ConfigManager ''' @Desc : CloudStackTestClient is encapsulated class for getting various \ @@ -48,6 +52,17 @@ class cloudstackTestClient(object): self.createDbConnection(dbSvrDetails.dbSvr, dbSvrDetails.port, dbSvrDetails.user, dbSvrDetails.passwd, dbSvrDetails.db) + ''' + Provides the Configuration Object to users through getConfigParser + The purpose of this object is to parse the config + and provide dictionary of the config so users can + use that configuration.Users can later call getConfig + on this object and it will return the default parsed + config dictionary from default configuration file, + they can overwrite it with providing their own + configuration file as well. + ''' + self.configObj = ConfigManager() self.asyncJobMgr = None self.id = None self.defaultWorkerThreads = defaultWorkerThreads @@ -154,6 +169,9 @@ class cloudstackTestClient(object): def getDbConnection(self): return self.dbConnection + def getConfigParser(self): + return self.configObj + def getApiClient(self): self.apiClient.id = self.identifier return self.apiClient diff --git a/tools/marvin/marvin/config/setup.cfg b/tools/marvin/marvin/config/setup.cfg new file mode 100644 index 00000000000..216314ff6bc --- /dev/null +++ b/tools/marvin/marvin/config/setup.cfg @@ -0,0 +1,228 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +{ + "zones": [ + { + "name": "Sandbox-simulator", + "guestcidraddress": "10.1.1.0/24", + "dns1": "10.147.28.6", + "physical_networks": [ + { + "broadcastdomainrange": "Zone", + "vlan": "100-200", + "name": "Sandbox-pnet", + "traffictypes": [ + { + "typ": "Guest" + }, + { + "typ": "Management" + }, + { + "typ": "Public" + } + ], + "providers": [ + { + "broadcastdomainrange": "ZONE", + "name": "VirtualRouter" + }, + { + "broadcastdomainrange": "ZONE", + "name": "VpcVirtualRouter" + }, + { + "broadcastdomainrange": "ZONE", + "name": "InternalLbVm" + } + ], + "isolationmethods": [ + "VLAN" + ] + } + ], + "ipranges": [ + { + "startip": "192.168.2.2", + "endip": "192.168.2.200", + "netmask": "255.255.255.0", + "vlan": "50", + "gateway": "192.168.2.1" + } + ], + "networktype": "Advanced", + "pods": [ + { + "endip": "172.16.15.200", + "name": "POD0", + "startip": "172.16.15.2", + "netmask": "255.255.255.0", + "clusters": [ + { + "clustername": "C0", + "hypervisor": "simulator", + "hosts": [ + { + "username": "root", + "url": "http://sim/c0/h0", + "password": "password" + }, + { + "username": "root", + "url": "http://sim/c0/h1", + "password": "password" + } + ], + "clustertype": "CloudManaged", + "primaryStorages": [ + { + "url": "nfs://10.147.28.6:/export/home/sandbox/primary0", + "name": "PS0" + }, + { + "url": "nfs://10.147.28.6:/export/home/sandbox/primary1", + "name": "PS1" + } + ] + }, + { + "clustername": "C1", + "hypervisor": "simulator", + "hosts": [ + { + "username": "root", + "url": "http://sim/c1/h0", + "password": "password" + } + ], + "clustertype": "CloudManaged", + "primaryStorages": [ + { + "url": "nfs://10.147.28.6:/export/home/sandbox/primary2", + "name": "PS2" + } + ] + } + ], + "gateway": "172.16.15.1" + } + ], + "internaldns1": "10.147.28.6", + "secondaryStorages": [ + { + "url": "nfs://10.147.28.6:/export/home/sandbox/secondary", + "provider" : "NFS" + } + ] + } + ], + "dbSvr": { + "dbSvr": "localhost", + "passwd": "cloud", + "db": "cloud", + "port": 3306, + "user": "cloud" + }, + "logger": [ + { + "name": "TestClient", + "file": "/tmp/testclient.log" + }, + { + "name": "TestCase", + "file": "/tmp/testcase.log" + } + ], + "globalConfig": [ + { + "name": "network.gc.wait", + "value": "60" + }, + { + "name": "storage.cleanup.interval", + "value": "300" + }, + { + "name": "vm.op.wait.interval", + "value": "5" + }, + { + "name": "default.page.size", + "value": "10000" + }, + { + "name": "network.gc.interval", + "value": "60" + }, + { + "name": "instance.name", + "value": "QA" + }, + { + "name": "workers", + "value": "10" + }, + { + "name": "account.cleanup.interval", + "value": "600" + }, + { + "name": "guest.domain.suffix", + "value": "sandbox.simulator" + }, + { + "name": "expunge.delay", + "value": "60" + }, + { + "name": "vm.allocation.algorithm", + "value": "random" + }, + { + "name": "expunge.interval", + "value": "60" + }, + { + "name": "expunge.workers", + "value": "3" + }, + { + "name": "check.pod.cidrs", + "value": "true" + }, + { + "name": "secstorage.allowed.internal.sites", + "value": "10.147.28.0/24" + }, + { + "name": "direct.agent.load.size", + "value": "1000" + } + ], + "mgtSvr": [ + { + "mgtSvrIp": "localhost", + "passwd": "password", + "user": "root", + "port": 8096, + "hypervisor": "simulator", + "useHttps": "False", + "certCAPath": "NA", + "certPath": "NA" + } + ] +} diff --git a/tools/marvin/marvin/configGenerator.py b/tools/marvin/marvin/configGenerator.py index 0cfad30569c..631e40fce43 100644 --- a/tools/marvin/marvin/configGenerator.py +++ b/tools/marvin/marvin/configGenerator.py @@ -19,6 +19,7 @@ import json import os from optparse import OptionParser import jsonHelper +from marvin.codes import * class managementServer(object): @@ -82,7 +83,7 @@ class zone(object): self.cacheStorages = [] -class traffictype(object): +class trafficType(object): def __init__(self, typ, labeldict=None): self.typ = typ # Guest/Management/Public if labeldict: @@ -153,7 +154,7 @@ class host(object): self.memory = None -class physical_network(object): +class physicalNetwork(object): def __init__(self): self.name = None self.tags = [] @@ -299,6 +300,126 @@ class bigip(object): for r in req]) +class ConfigManager(object): + + ''' + @Name: configManager + @Desc: 1. It provides the basic configuration facilities to marvin. + 2. User can just add configuration files for his tests, deployment + etc, under one config folder before running their tests. + cs/tools/marvin/marvin/config. + They can remove all hard coded values from code and separate + it out as config at this location. + Either add this to the existing setup.cfg as separate section + or add new configuration. + 3. This will thus removes hard coded tests and separate + data from tests. + 4. This API is provided as an additional facility under + cloudstackTestClient and users can get the + configuration object as similar to apiclient,dbconnection + etc to drive their test. + 5. They just add their configuration for a test, + setup etc,at one single place under configuration dir + and use "getConfigParser" API of cloudstackTestClient + It will give them "configObj".They can either pass their own + config file for parsing to "getConfig" or it will use + default config file @ config/setup.cfg. + 6. They will then get the dictionary of parsed + configuration and can use it further to drive their tests or + config drive + 7. Test features, can drive their setups thus removing hard coded + values. Configuration default file will be under config and as + setup.cfg. + 8. Users can use their own configuration file passed to + "getConfig" API,once configObj is returned. + ''' + + def __init__(self): + self.filePath = "config/config.cfg" + self.parsedDict = None + if self.__verifyFile(self.filePath) is not False: + self.parsedDict = self.__parseConfig(self.filePath) + + def __parseConfig(self, file): + ''' + @Name : __parseConfig + @Description: Parses the Input configuration Json file + and returns a dictionary from the file. + @Input : NA + @Output : Returns the parsed dictionary from json file + Returns None for invalid input or if parsing failed + ''' + config_dict = None + try: + configlines = [] + with open(file, 'r') as fp: + for line in fp: + if len(line) != 0: + ws = line.strip() + if ws[0] not in ["#"]: + configlines.append(ws) + config_dict = json.loads("\n".join(configlines)) + except Exception, e: + #Will replace with log once we have logging done + print "\n Exception occurred under __parseConfig", e + finally: + return config_dict + + def __verifyFile(self, file): + ''' + @Name : __parseConfig + @Description: Parses the Input configuration Json file + and returns a dictionary from the file. + @Input : file NA + @Output : True or False based upon file input validity + and availability + ''' + if file is None or file == '': + return False + if os.path.exists(file) is False: + return False + return True + + def __getSectionData(self, return_dict, section=None): + ''' + @Name: getSectionData + @Desc: Gets the Section data of a particular section + under parsed dictionary + @Input: Parsed Dictionary from configuration file + section to be returned from this dict + @Output:Section matching inside the parsed data + ''' + if return_dict is not None: + inp = return_dict + elif self.parsedDict is None: + return INVALID_INPUT + else: + inp = self.parsedDict + + if section is not None: + return inp.get(section) + else: + return inp + + def getConfig(self, file_path=None, section=None): + ''' + @Name: getConfig + @Desc : Parses and converts the given configuration file to dictionary + @Input : file_path: path where the configuration needs to be passed + section: specific section inside the file + @Output: INVALID_INPUT: This value is returned if the input + is invalid or not able to be parsed + Parsed configuration dictionary from json file + ''' + ret = None + if file not in [None, '']: + if self.__verifyFile(file_path) is False: + return INVALID_INPUT + else: + ret = self.__parseConfig(file_path) + return self.__getSectionData(ret, section) + + def getDeviceUrl(obj): req = zip(obj.__dict__.keys(), obj.__dict__.values()) if obj.hostname: @@ -309,7 +430,7 @@ def getDeviceUrl(obj): return None -def describe_setup_in_basic_mode(): +def descSetupInBasicMode(): '''sample code to generate setup configuration file''' zs = cloudstackConfiguration() @@ -328,9 +449,9 @@ def describe_setup_in_basic_mode(): sgprovider.broadcastdomainrange = 'Pod' sgprovider.name = 'SecurityGroupProvider' - pn = physical_network() + pn = physicalNetwork() pn.name = "test-network" - pn.traffictypes = [traffictype("Guest"), traffictype("Management")] + pn.traffictypes = [trafficType("Guest"), trafficType("Management")] pn.providers.append(sgprovider) z.physical_networks.append(pn) @@ -428,7 +549,7 @@ def describe_setup_in_basic_mode(): return zs -def describe_setup_in_eip_mode(): +def descSetupInEipMode(): """ Setting up an EIP/ELB enabled zone with netscaler provider """ @@ -462,12 +583,12 @@ def describe_setup_in_eip_mode(): ns.hostname = '10.147.40.100' nsprovider.devices.append(ns) - pn = physical_network() + pn = physicalNetwork() pn.name = "test-network" - pn.traffictypes = [traffictype("Guest", + pn.traffictypes = [trafficType("Guest", {"xen": "cloud-guest"}), - traffictype("Management"), - traffictype("Public", {"xen": "cloud-public"})] + trafficType("Management"), + trafficType("Public", {"xen": "cloud-public"})] pn.providers.extend([sgprovider, nsprovider]) z.physical_networks.append(pn) @@ -563,7 +684,7 @@ def describe_setup_in_eip_mode(): return zs -def describe_setup_in_advanced_mode(): +def descSetupInAdvancedMode(): '''sample code to generate setup configuration file''' zs = cloudstackConfiguration() @@ -578,10 +699,10 @@ def describe_setup_in_advanced_mode(): z.guestcidraddress = "10.1.1.0/24" z.vlan = "100-2000" - pn = physical_network() + pn = physicalNetwork() pn.name = "test-network" - pn.traffictypes = [traffictype("Guest"), traffictype("Management"), - traffictype("Public")] + pn.traffictypes = [trafficType("Guest"), trafficType("Management"), + trafficType("Public")] vpcprovider = provider('VpcVirtualRouter') @@ -697,7 +818,7 @@ def describe_setup_in_advanced_mode(): '''sample code to generate setup configuration file''' -def describe_setup_in_advancedsg_mode(): +def descSetupInAdvancedsgMode(): zs = cloudstackConfiguration() for l in range(1): @@ -711,9 +832,9 @@ def describe_setup_in_advancedsg_mode(): z.vlan = "100-2000" z.securitygroupenabled = "true" - pn = physical_network() + pn = physicalNetwork() pn.name = "test-network" - pn.traffictypes = [traffictype("Guest"), traffictype("Management")] + pn.traffictypes = [trafficType("Guest"), trafficType("Management")] #If security groups are reqd sgprovider = provider() @@ -864,10 +985,10 @@ by default is ./datacenterCfg") if options.inputfile: config = getSetupConfig(options.inputfile) if options.advanced: - config = describe_setup_in_advanced_mode() + config = descSetupInAdvancedMode() elif options.advancedsg: - config = describe_setup_in_advancedsg_mode() + config = descSetupInAdvancedsgMode() else: - config = describe_setup_in_basic_mode() + config = descSetupInBasicMode() generate_setup_config(config, options.output) diff --git a/tools/marvin/marvin/integration/lib/utils.py b/tools/marvin/marvin/integration/lib/utils.py index 7d662af265c..b6e38ec9e6f 100644 --- a/tools/marvin/marvin/integration/lib/utils.py +++ b/tools/marvin/marvin/integration/lib/utils.py @@ -323,7 +323,7 @@ def is_snapshot_on_nfs(apiclient, dbconn, config, zoneid, snapshotid): def validateList(inp): ''' - @name: validateList + @name: validateList @Description: 1. A utility function to validate whether the input passed is a list 2. The list is empty or not