From 1c9cd9d36077331a7b25d19e18ae73918e5662f6 Mon Sep 17 00:00:00 2001 From: Prasanna Santhanam Date: Fri, 7 Jun 2013 14:49:00 +0530 Subject: [PATCH 01/30] CLOUDSTACK-2884: str object interpreted as index Casting to an int for almostEqual comparison Signed-off-by: Prasanna Santhanam --- test/integration/smoke/test_service_offerings.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/integration/smoke/test_service_offerings.py b/test/integration/smoke/test_service_offerings.py index a56e34d2874..0213c04cb02 100644 --- a/test/integration/smoke/test_service_offerings.py +++ b/test/integration/smoke/test_service_offerings.py @@ -92,7 +92,7 @@ class Services: "displaytext": "Small Instance", "cpunumber": 1, "cpuspeed": 100, - "memory": 256, + "memory": 128, }, "medium": { @@ -433,7 +433,7 @@ class TestServiceOfferings(cloudstackTestCase): ) self.assertAlmostEqual( int(total_mem) / 1024, # In MBs - self.small_offering.memory, + int(self.small_offering.memory), "Check Memory(kb) for small offering" ) return From 302741d309f335c1f8c3a36286bec513fd9ed2f4 Mon Sep 17 00:00:00 2001 From: Prasanna Santhanam Date: Fri, 7 Jun 2013 15:02:55 +0530 Subject: [PATCH 02/30] Log the actual NAT ip attempting to SSH into VM IP is the guest IP and it's the NAT ip that is used for ssh. Signed-off-by: Prasanna Santhanam --- test/integration/smoke/test_network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/smoke/test_network.py b/test/integration/smoke/test_network.py index eb33c630ccf..a65748d2e97 100644 --- a/test/integration/smoke/test_network.py +++ b/test/integration/smoke/test_network.py @@ -1209,7 +1209,7 @@ class TestRebootRouter(cloudstackTestCase): except Exception as e: self.fail( "SSH Access failed for %s: %s" % \ - (self.vm_1.ipaddress, e)) + (self.nat_rule.ipaddress.ipaddress, e)) return def tearDown(self): From e17a0c23b8d4d7cbbcbf4a95cf545b0d8ae59aaa Mon Sep 17 00:00:00 2001 From: Sebastien Goasguen Date: Fri, 7 Jun 2013 06:03:42 -0400 Subject: [PATCH 03/30] [GSoC]: Proposal from Dharmesh Kakadia --- docs/en-US/CloudStack_GSoC_Guide.xml | 1 + docs/en-US/gsoc-dharmesh.xml | 149 +++++++++++++++++++ docs/en-US/images/mesos-integration-arch.jpg | 0 3 files changed, 150 insertions(+) create mode 100644 docs/en-US/gsoc-dharmesh.xml create mode 100644 docs/en-US/images/mesos-integration-arch.jpg diff --git a/docs/en-US/CloudStack_GSoC_Guide.xml b/docs/en-US/CloudStack_GSoC_Guide.xml index b7ba61f8ee4..243a0ca361b 100644 --- a/docs/en-US/CloudStack_GSoC_Guide.xml +++ b/docs/en-US/CloudStack_GSoC_Guide.xml @@ -48,6 +48,7 @@ + diff --git a/docs/en-US/gsoc-dharmesh.xml b/docs/en-US/gsoc-dharmesh.xml new file mode 100644 index 00000000000..5e2bf734d7f --- /dev/null +++ b/docs/en-US/gsoc-dharmesh.xml @@ -0,0 +1,149 @@ + + +%BOOK_ENTITIES; +]> + + + + + Dharmesh's 2013 GSoC Proposal + This chapter describes Dharmrsh's 2013 Google Summer of Code project within the &PRODUCT; ASF project. It is a copy paste of the submitted proposal. +
+ Abstract + + The project aims to bring cloudformation like service to cloudstack. One of the prime use-case is cluster computing frameworks on cloudstack. A cloudformation service will give users and administrators of cloudstack ability to manage and control a set of resources easily. The cloudformation will allow booting and configuring a set of VMs and form a cluster. Simple example would be LAMP stack. More complex clusters such as mesos or hadoop cluster requires a little more advanced configuration. There is already some work done by Chiradeep Vittal at this front [5]. In this project, I will implement server side cloudformation service for cloudstack and demonstrate how to run mesos cluster using it. + +
+ +
+ Mesos + + Mesos is a resource management platform for clusters. It aims to increase resource utilization of clusters by sharing cluster resources among multiple processing frameworks(like MapReduce, MPI, Graph Processing) or multiple instances of same framework. It provides efficient resource isolation through use of containers. Uses zookeeper for state maintenance and fault tolerance. + +
+ +
+ What can run on mesos ? + + Spark: A cluster computing framework based on the Resilient Distributed Datasets (RDDs) abstraction. RDD is more generalized than MapReduce and can support iterative and interactive computation while retaining fault tolerance, scalability, data locality etc. + + Hadoop:: Hadoop is fault tolerant and scalable distributed computing framework based on MapReduce abstraction. + + Begel:: A graph processing framework based on pregel. + + and other frameworks like MPI, Hypertable. +
+ +
+ How to deploy mesos ? + + Mesos provides cluster installation scripts for cluster deployment. There are also scripts available to deploy a cluster on Amazon EC2. It would be interesting to see if this scripts can be leveraged in anyway. +
+ +
+ Deliverables + + + Deploy CloudStack and understand instance configuration/contextualization + + + Test and deploy Mesos on a set of CloudStack based VM, manually. Design/propose an automation framework + + + Test stackmate and engage chiradeep (report bugs, make suggestion, make pull request) + + + Create cloudformation template to provision a Mesos Cluster + + + Compare with Apache Whirr or other cluster provisioning tools for server side implementation of cloudformation service. + + +
+ +
+ Architecture and Tools + + The high level architecture is as follows: + + + + + + + + + + + It includes following components: + + + + CloudFormation Query API server: + This acts as a point of contact to and exposes CloudFormation functionality as Query API. This can be accessed directly or through existing tools from Amazon AWS for their cloudformation service. It will be easy to start as a module which resides outside cloudstack at first and I plan to use dropwizard [3] to start with. Later may be the API server can be merged with cloudstack core. I plan to use mysql for storing details of clusters. + + + + Provisioning: + + Provisioning module is responsible for handling the booting process of the VMs through cloudstack. This uses the cloudstack APIs for launching VMs. I plan to use preconfigured templates/images with required dependencies installed, which will make cluster creation process much faster even for large clusters. Error handling is very important part of this module. For example, what you do if few VMs fail to boot in cluster ? + + + + Configuration: + + This module deals with configuring the VMs to form a cluster. This can be done via manual scripts/code or via configuration management tools like chef/ironfan/knife. Potentially workflow automation tools like rundeck [4] also can be used. Also Apache whirr and Provisionr are options. I plan to explore this tools and select suitable ones. + + + +
+ +
+ API + + Query API will be based on Amazon AWS cloudformation service. This will allow leveraging existing tools for AWS. +
+ +
+ Timeline + 1-1.5 week : project design. Architecture, tools selection, API design + 1-1.5 week : getting familiar with cloudstack and stackmate codebase and architecture details + 1-1.5 week : getting familiar with mesos internals + 1-1.5 week : setting up the dev environment and create mesos templates + 2-3 week : build provisioning and configuration module + Midterm evaluation: provisioning module, configuration module + 2-3 week : develope cloudformation server side implementation + 2-3 week : test and integrate +
+ +
+ Future Work + + + Auto Scaling: + Automatically adding or removing VMs from mesos cluster based on various conditions like utilization going above/below a static threshold. There can be more sophisticated strategies based on prediction or fine grained metric collection with tight integration with mesos framework. + + + Cluster Simulator: + Integrating with existing simulator to simulate mesos clusters. This can be useful in various scenarios, for example while developing a new scheduling algorithm, testing autoscaling etc. + + +
+ diff --git a/docs/en-US/images/mesos-integration-arch.jpg b/docs/en-US/images/mesos-integration-arch.jpg new file mode 100644 index 00000000000..e69de29bb2d From bcc5baa1636037baa8c5ffbd3bbf70df8af4d024 Mon Sep 17 00:00:00 2001 From: Pranav Saxena Date: Fri, 7 Jun 2013 16:23:44 +0530 Subject: [PATCH 04/30] CLOUDSTACK-869:Netscaler support as an external LB provider:front end --- ui/scripts/network.js | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/ui/scripts/network.js b/ui/scripts/network.js index 050b75c49ff..a3e239be6dc 100755 --- a/ui/scripts/network.js +++ b/ui/scripts/network.js @@ -4531,7 +4531,21 @@ networkdomain: { docID: 'helpVPCDomain', label: 'label.DNS.domain.for.guest.networks' + }, + + loadbalancer:{ //Support for Netscaler as an external device for load balancing + label:'Load Balancer', + select:function(args){ + + var items = []; + items.push({id:'vpcvirtualrouter' , description:'VPC Virtual Router'}); + items.push({id:'netscaler' , description:'NetScaler'}); + + args.response.success({data:items}); } + + } + } }, action: function(args) { From 5233e3216b11e69c7c7e051f0a6c3d0c6bf98803 Mon Sep 17 00:00:00 2001 From: Pranav Saxena Date: Fri, 7 Jun 2013 16:53:32 +0530 Subject: [PATCH 05/30] CLOUDSTACK-869:Netscaler support as an external LB provider --- ui/scripts/network.js | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/ui/scripts/network.js b/ui/scripts/network.js index a3e239be6dc..9e60cbcc20f 100755 --- a/ui/scripts/network.js +++ b/ui/scripts/network.js @@ -4536,20 +4536,29 @@ loadbalancer:{ //Support for Netscaler as an external device for load balancing label:'Load Balancer', select:function(args){ - - var items = []; - items.push({id:'vpcvirtualrouter' , description:'VPC Virtual Router'}); - items.push({id:'netscaler' , description:'NetScaler'}); - + $.ajax({ + url:createURL('listVPCOfferings&listall=true'), + dataType:'json', + success:function(json){ + var items=[]; + var vpcObj = json.listvpcofferingsresponse.vpcoffering; + $(vpcObj).each(function(){ + items.push({id:this.id , description:this.name}); + }); args.response.success({data:items}); - } + + } + + }); + + } } } }, action: function(args) { - var defaultvpcofferingid; + /* var defaultvpcofferingid; $.ajax({ url: createURL("listVPCOfferings"), dataType: "json", @@ -4560,14 +4569,14 @@ success: function(json) { defaultvpcofferingid = json.listvpcofferingsresponse.vpcoffering[0].id; } - }); + });*/ var dataObj = { name: args.data.name, displaytext: args.data.displaytext, zoneid: args.data.zoneid, cidr: args.data.cidr, - vpcofferingid: defaultvpcofferingid + vpcofferingid: args.data.loadbalancer // Support for external load balancer }; if(args.data.networkdomain != null && args.data.networkdomain.length > 0) From 5dc7387d3b6b1abd841abc92e3c76a3894213d82 Mon Sep 17 00:00:00 2001 From: Saksham Srivastava Date: Wed, 5 Jun 2013 16:28:05 +0530 Subject: [PATCH 06/30] CLOUDSTACK-1647: IP Reservation should not happen if the guest-vm cidr and network cidr is not same but their start ip and end ip are same. Signed-off-by: Sateesh Chodapuneedi --- .../com/cloud/network/NetworkServiceImpl.java | 15 ++++++++++ utils/src/com/cloud/utils/net/NetUtils.java | 28 +++++++++++++++++++ .../com/cloud/utils/net/NetUtilsTest.java | 19 +++++++++++++ 3 files changed, 62 insertions(+) diff --git a/server/src/com/cloud/network/NetworkServiceImpl.java b/server/src/com/cloud/network/NetworkServiceImpl.java index 29a36b9fe5f..c2af8e8fe60 100755 --- a/server/src/com/cloud/network/NetworkServiceImpl.java +++ b/server/src/com/cloud/network/NetworkServiceImpl.java @@ -2116,6 +2116,21 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService { } } + // In some scenarios even though guesVmCidr and network CIDR do not appear similar but + // the IP ranges exactly matches, in these special cases make sure no Reservation gets applied + if (network.getNetworkCidr() == null) { + if (NetUtils.isSameIpRange(guestVmCidr, network.getCidr()) && !guestVmCidr.equals(network.getCidr())) { + throw new InvalidParameterValueException("The Start IP and End IP of guestvmcidr: "+ guestVmCidr + " and CIDR: " + network.getCidr() + " are same, " + + "even though both the cidrs appear to be different. As a precaution no IP Reservation will be applied."); + } + } else { + if(NetUtils.isSameIpRange(guestVmCidr, network.getNetworkCidr()) && !guestVmCidr.equals(network.getNetworkCidr())) { + throw new InvalidParameterValueException("The Start IP and End IP of guestvmcidr: "+ guestVmCidr + " and Network CIDR: " + network.getNetworkCidr() + " are same, " + + "even though both the cidrs appear to be different. As a precaution IP Reservation will not be affected. If you want to reset IP Reservation, " + + "specify guestVmCidr to be: " + network.getNetworkCidr()); + } + } + // When reservation is applied for the first time, network_cidr will be null // Populate it with the actual network cidr if (network.getNetworkCidr() == null) { diff --git a/utils/src/com/cloud/utils/net/NetUtils.java b/utils/src/com/cloud/utils/net/NetUtils.java index 8c094c85088..12ac3e6eb08 100755 --- a/utils/src/com/cloud/utils/net/NetUtils.java +++ b/utils/src/com/cloud/utils/net/NetUtils.java @@ -1023,6 +1023,34 @@ public class NetUtils { return NetUtils.getIpRangeStartIpFromCidr(splitResult[0], size); } + // Check if 2 CIDRs have exactly same IP Range + public static boolean isSameIpRange (String cidrA, String cidrB) { + + if(!NetUtils.isValidCIDR(cidrA)) { + s_logger.info("Invalid value of cidr " + cidrA); + return false; + } + if (!NetUtils.isValidCIDR(cidrB)) { + s_logger.info("Invalid value of cidr " + cidrB); + return false; + } + String[] cidrPairFirst = cidrA.split("\\/"); + String[] cidrPairSecond = cidrB.split("\\/"); + + Long networkSizeFirst = Long.valueOf(cidrPairFirst[1]); + Long networkSizeSecond = Long.valueOf(cidrPairSecond[1]); + String ipRangeFirst [] = NetUtils.getIpRangeFromCidr(cidrPairFirst[0], networkSizeFirst); + String ipRangeSecond [] = NetUtils.getIpRangeFromCidr(cidrPairFirst[0], networkSizeSecond); + + long startIpFirst = NetUtils.ip2Long(ipRangeFirst[0]); + long endIpFirst = NetUtils.ip2Long(ipRangeFirst[1]); + long startIpSecond = NetUtils.ip2Long(ipRangeSecond[0]); + long endIpSecond = NetUtils.ip2Long(ipRangeSecond[1]); + if(startIpFirst == startIpSecond && endIpFirst == endIpSecond) { + return true; + } + return false; + } public static boolean validateGuestCidr(String cidr) { // RFC 1918 - The Internet Assigned Numbers Authority (IANA) has reserved the // following three blocks of the IP address space for private internets: diff --git a/utils/test/com/cloud/utils/net/NetUtilsTest.java b/utils/test/com/cloud/utils/net/NetUtilsTest.java index 16d3402f0e6..9952d3cd864 100644 --- a/utils/test/com/cloud/utils/net/NetUtilsTest.java +++ b/utils/test/com/cloud/utils/net/NetUtilsTest.java @@ -136,4 +136,23 @@ public class NetUtilsTest extends TestCase { assertTrue(NetUtils.getPrimaryPvlanFromUri(uri).equals("123")); assertTrue(NetUtils.getIsolatedPvlanFromUri(uri).equals("456")); } + + public void testIsSameIpRange() { + //Test to check IP Range of 2 CIDRs + String cidrFirst = "10.0.144.0/20"; + String cidrSecond = "10.0.151.0/20"; + String cidrThird = "10.0.144.0/21"; + assertTrue(NetUtils.isValidCIDR(cidrFirst)); + assertTrue(NetUtils.isValidCIDR(cidrSecond)); + assertTrue(NetUtils.isValidCIDR(cidrThird)); + + //Check for exactly same CIDRs + assertTrue(NetUtils.isSameIpRange(cidrFirst, cidrFirst)); + //Check for 2 different CIDRs, but same IP Range + assertTrue(NetUtils.isSameIpRange(cidrFirst, cidrSecond)); + //Check for 2 different CIDRs and different IP Range + assertFalse(NetUtils.isSameIpRange(cidrFirst, cidrThird)); + //Check for Incorrect format of CIDR + assertFalse(NetUtils.isSameIpRange(cidrFirst, "10.3.6.5/50")); + } } From 195e823b10c5519f90aefa74a0cb104f0ff57b07 Mon Sep 17 00:00:00 2001 From: Prasanna Santhanam Date: Fri, 7 Jun 2013 16:23:06 +0530 Subject: [PATCH 07/30] CLOUDSTACK-2882: Fix the diskdevice based on hypervisor KVM > /dev/vda Xen > /dev/xvdd Signed-off-by: Prasanna Santhanam --- test/integration/smoke/test_vm_life_cycle.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/integration/smoke/test_vm_life_cycle.py b/test/integration/smoke/test_vm_life_cycle.py index 8c78149addf..afe9b8a6331 100644 --- a/test/integration/smoke/test_vm_life_cycle.py +++ b/test/integration/smoke/test_vm_life_cycle.py @@ -730,6 +730,15 @@ class TestVMLifeCycle(cloudstackTestCase): cmd.virtualmachineid = self.virtual_machine.id self.apiclient.attachIso(cmd) + #determine device type from hypervisor + hosts = Host.list(self.apiclient, id=self.virtual_machine.hostid) + self.assertTrue(isinstance(hosts, list)) + self.assertTrue(len(hosts) > 0) + self.debug("Found %s host" % hosts[0].hypervisor) + + if hosts[0].hypervisor.lower() == "kvm": + self.services["diskdevice"] = "/dev/vda" + try: ssh_client = self.virtual_machine.get_ssh_client() except Exception as e: From da5c4619c394eaedec55c277bc5e71de379d6600 Mon Sep 17 00:00:00 2001 From: Sanjay Tripathi Date: Fri, 7 Jun 2013 17:08:31 +0530 Subject: [PATCH 08/30] CLOUDSTACK-2286: Volume created from snapshot state is in allocated state instead of Ready state which is letting Primary storage not to increment the resources. --- .../storage/volume/VolumeServiceImpl.java | 79 ++++++++++--------- 1 file changed, 40 insertions(+), 39 deletions(-) diff --git a/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java b/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java index 7fdf6bb18d3..54dcbd2599e 100644 --- a/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java +++ b/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java @@ -428,51 +428,52 @@ public class VolumeServiceImpl implements VolumeService { public AsyncCallFuture createVolumeFromSnapshot( VolumeInfo volume, DataStore store, SnapshotInfo snapshot) { AsyncCallFuture future = new AsyncCallFuture(); - + try { - DataObject volumeOnStore = store.create(volume); - volume.processEvent(Event.CreateOnlyRequested); - CreateVolumeFromBaseImageContext context = new CreateVolumeFromBaseImageContext(null, - (VolumeObject)volume, store, volumeOnStore, future); - AsyncCallbackDispatcher caller = AsyncCallbackDispatcher.create(this); - caller.setCallback(caller.getTarget().createVolumeFromSnapshotCallback(null, null)) - .setContext(context); - this.motionSrv.copyAsync(snapshot, volumeOnStore, caller); + DataObject volumeOnStore = store.create(volume); + volume = this.volFactory.getVolume(volume.getId(), store); + volume.processEvent(Event.CreateOnlyRequested); + CreateVolumeFromBaseImageContext context = new CreateVolumeFromBaseImageContext(null, + (VolumeObject)volume, store, volumeOnStore, future); + AsyncCallbackDispatcher caller = AsyncCallbackDispatcher.create(this); + caller.setCallback(caller.getTarget().createVolumeFromSnapshotCallback(null, null)) + .setContext(context); + this.motionSrv.copyAsync(snapshot, volumeOnStore, caller); } catch (Exception e) { - s_logger.debug("create volume from snapshot failed", e); - VolumeApiResult result = new VolumeApiResult(volume); - result.setResult(e.toString()); - future.complete(result); + s_logger.debug("create volume from snapshot failed", e); + VolumeApiResult result = new VolumeApiResult(volume); + result.setResult(e.toString()); + future.complete(result); } - + return future; } - - protected Void createVolumeFromSnapshotCallback(AsyncCallbackDispatcher callback, - CreateVolumeFromBaseImageContext context) { - CopyCommandResult result = callback.getResult(); - VolumeInfo volume = context.vo; - VolumeApiResult apiResult = new VolumeApiResult(volume); - Event event = null; - if (result.isFailed()) { - apiResult.setResult(result.getResult()); - event = Event.OperationFailed; - } else { - event = Event.OperationSuccessed; - } - - try { - volume.processEvent(event); - } catch (Exception e) { - s_logger.debug("create volume from snapshot failed", e); - apiResult.setResult(e.toString()); - } - - AsyncCallFuture future = context.future; - future.complete(apiResult); - return null; + + protected Void createVolumeFromSnapshotCallback(AsyncCallbackDispatcher callback, + CreateVolumeFromBaseImageContext context) { + CopyCommandResult result = callback.getResult(); + VolumeInfo volume = context.vo; + VolumeApiResult apiResult = new VolumeApiResult(volume); + Event event = null; + if (result.isFailed()) { + apiResult.setResult(result.getResult()); + event = Event.OperationFailed; + } else { + event = Event.OperationSuccessed; + } + + try { + volume.processEvent(event); + } catch (Exception e) { + s_logger.debug("create volume from snapshot failed", e); + apiResult.setResult(e.toString()); + } + + AsyncCallFuture future = context.future; + future.complete(apiResult); + return null; } - + protected VolumeVO duplicateVolumeOnAnotherStorage(Volume volume, StoragePool pool) { Long lastPoolId = volume.getPoolId(); VolumeVO newVol = new VolumeVO(volume); From ce17f856eacce32e09b43dd955d5e2da2a678766 Mon Sep 17 00:00:00 2001 From: Brian Federle Date: Fri, 7 Jun 2013 11:05:01 -0700 Subject: [PATCH 09/30] CLOUDSTACK-2811: Default add primary storage dialog to scope=cluster --- ui/scripts/system.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ui/scripts/system.js b/ui/scripts/system.js index e3d7b4172fa..f853ed53f7e 100644 --- a/ui/scripts/system.js +++ b/ui/scripts/system.js @@ -11425,8 +11425,8 @@ label: 'label.scope', select: function(args) { var scope = [ - { id: 'zone', description: _l('label.zone.wide') }, - { id: 'cluster', description: _l('label.cluster') } + { id: 'cluster', description: _l('label.cluster') }, + { id: 'zone', description: _l('label.zone.wide') } // { id: 'host', description: _l('label.host') } ]; From 8c9f681f9eba377d1eee279329090fbbdc46c5ab Mon Sep 17 00:00:00 2001 From: Prachi Damle Date: Fri, 7 Jun 2013 15:27:39 -0700 Subject: [PATCH 10/30] CLOUDSTACK-2350: Anti-Affinity - As admin user, when tryinto update the affinity group for a Vm that is deployed by a regular user , he is presented with admin's affinity groups. Changes: - listAffinityGroups API takes in accountname and domainId parameter - For admin, listall=true should return all affinity groups of all users --- .../affinitygroup/ListAffinityGroupsCmd.java | 7 ++-- .../apache/cloudstack/query/QueryService.java | 3 +- .../com/cloud/api/query/QueryManagerImpl.java | 33 +++++++++++-------- .../com/cloud/user/AccountManagerImpl.java | 4 +++ 4 files changed, 29 insertions(+), 18 deletions(-) diff --git a/api/src/org/apache/cloudstack/api/command/user/affinitygroup/ListAffinityGroupsCmd.java b/api/src/org/apache/cloudstack/api/command/user/affinitygroup/ListAffinityGroupsCmd.java index 9310fb91016..d966a4c28b2 100644 --- a/api/src/org/apache/cloudstack/api/command/user/affinitygroup/ListAffinityGroupsCmd.java +++ b/api/src/org/apache/cloudstack/api/command/user/affinitygroup/ListAffinityGroupsCmd.java @@ -19,7 +19,7 @@ package org.apache.cloudstack.api.command.user.affinitygroup; import org.apache.cloudstack.affinity.AffinityGroupResponse; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiConstants; -import org.apache.cloudstack.api.BaseListCmd; +import org.apache.cloudstack.api.BaseListAccountResourcesCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.response.ListResponse; import org.apache.cloudstack.api.response.UserVmResponse; @@ -28,7 +28,7 @@ import org.apache.log4j.Logger; import com.cloud.async.AsyncJob; @APICommand(name = "listAffinityGroups", description = "Lists affinity groups", responseObject = AffinityGroupResponse.class) -public class ListAffinityGroupsCmd extends BaseListCmd { +public class ListAffinityGroupsCmd extends BaseListAccountResourcesCmd { public static final Logger s_logger = Logger.getLogger(ListAffinityGroupsCmd.class.getName()); private static final String s_name = "listaffinitygroupsresponse"; @@ -77,7 +77,8 @@ public class ListAffinityGroupsCmd extends BaseListCmd { public void execute(){ ListResponse response = _queryService.listAffinityGroups(id, affinityGroupName, - affinityGroupType, virtualMachineId, this.getStartIndex(), this.getPageSizeVal()); + affinityGroupType, virtualMachineId, this.getAccountName(), this.getDomainId(), this.isRecursive(), + this.listAll(), this.getStartIndex(), this.getPageSizeVal()); response.setResponseName(getCommandName()); this.setResponseObject(response); diff --git a/api/src/org/apache/cloudstack/query/QueryService.java b/api/src/org/apache/cloudstack/query/QueryService.java index 2dfd97cfa98..73e393b069f 100644 --- a/api/src/org/apache/cloudstack/query/QueryService.java +++ b/api/src/org/apache/cloudstack/query/QueryService.java @@ -86,7 +86,8 @@ public interface QueryService { public ListResponse listDataCenters(ListZonesByCmd cmd); public ListResponse listAffinityGroups(Long affinityGroupId, String affinityGroupName, - String affinityGroupType, Long vmId, Long startIndex, Long pageSize); + String affinityGroupType, Long vmId, String accountName, Long domainId, boolean isRecursive, + boolean listAll, Long startIndex, Long pageSize); public List listResource(ListResourceDetailsCmd cmd); diff --git a/server/src/com/cloud/api/query/QueryManagerImpl.java b/server/src/com/cloud/api/query/QueryManagerImpl.java index 28aecfc223d..beda75e863c 100644 --- a/server/src/com/cloud/api/query/QueryManagerImpl.java +++ b/server/src/com/cloud/api/query/QueryManagerImpl.java @@ -2401,9 +2401,10 @@ public class QueryManagerImpl extends ManagerBase implements QueryService { @Override public ListResponse listAffinityGroups(Long affinityGroupId, String affinityGroupName, - String affinityGroupType, Long vmId, Long startIndex, Long pageSize) { + String affinityGroupType, Long vmId, String accountName, Long domainId, boolean isRecursive, + boolean listAll, Long startIndex, Long pageSize) { Pair, Integer> result = listAffinityGroupsInternal(affinityGroupId, - affinityGroupName, affinityGroupType, vmId, startIndex, pageSize); + affinityGroupName, affinityGroupType, vmId, accountName, domainId, isRecursive, listAll, startIndex, pageSize); ListResponse response = new ListResponse(); List agResponses = ViewResponseHelper.createAffinityGroupResponses(result.first()); response.setResponses(agResponses, result.second()); @@ -2412,12 +2413,12 @@ public class QueryManagerImpl extends ManagerBase implements QueryService { public Pair, Integer> listAffinityGroupsInternal(Long affinityGroupId, - String affinityGroupName, String affinityGroupType, Long vmId, Long startIndex, Long pageSize) { + String affinityGroupName, String affinityGroupType, Long vmId, String accountName, Long domainId, + boolean isRecursive, boolean listAll, Long startIndex, Long pageSize) { Account caller = UserContext.current().getCaller(); Long accountId = caller.getAccountId(); - Long domainId = caller.getDomainId(); if (vmId != null) { UserVmVO userVM = _userVmDao.findById(vmId); @@ -2429,20 +2430,25 @@ public class QueryManagerImpl extends ManagerBase implements QueryService { return listAffinityGroupsByVM(vmId.longValue(), startIndex, pageSize); } + List permittedAccounts = new ArrayList(); + Ternary domainIdRecursiveListProject = new Ternary( + domainId, isRecursive, null); + _accountMgr.buildACLSearchParameters(caller, affinityGroupId, accountName, null, permittedAccounts, + domainIdRecursiveListProject, listAll, true); + domainId = domainIdRecursiveListProject.first(); + isRecursive = domainIdRecursiveListProject.second(); + ListProjectResourcesCriteria listProjectResourcesCriteria = domainIdRecursiveListProject.third(); + Filter searchFilter = new Filter(AffinityGroupJoinVO.class, "id", true, startIndex, pageSize); SearchBuilder groupSearch = _affinityGroupJoinDao.createSearchBuilder(); + _accountMgr.buildACLViewSearchBuilder(groupSearch, domainId, isRecursive, permittedAccounts, + listProjectResourcesCriteria); + groupSearch.select(null, Func.DISTINCT, groupSearch.entity().getId()); // select // distinct SearchCriteria sc = groupSearch.create(); - - if (accountId != null) { - sc.addAnd("accountId", SearchCriteria.Op.EQ, accountId); - } - - if (domainId != null) { - sc.addAnd("domainId", SearchCriteria.Op.EQ, domainId); - } + _accountMgr.buildACLViewSearchCriteria(sc, domainId, isRecursive, permittedAccounts, listProjectResourcesCriteria); if (affinityGroupId != null) { sc.addAnd("id", SearchCriteria.Op.EQ, affinityGroupId); @@ -2457,8 +2463,7 @@ public class QueryManagerImpl extends ManagerBase implements QueryService { } - Pair, Integer> uniqueGroupsPair = _affinityGroupJoinDao.searchAndCount(sc, - searchFilter); + Pair, Integer> uniqueGroupsPair = _affinityGroupJoinDao.searchAndCount(sc, searchFilter); // search group details by ids Integer count = uniqueGroupsPair.second(); if (count.intValue() == 0) { diff --git a/server/src/com/cloud/user/AccountManagerImpl.java b/server/src/com/cloud/user/AccountManagerImpl.java index 0d89639e31e..6b4bf0ed6b8 100755 --- a/server/src/com/cloud/user/AccountManagerImpl.java +++ b/server/src/com/cloud/user/AccountManagerImpl.java @@ -2208,6 +2208,10 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M domainIdRecursiveListProject.second(true); } } + } else if (domainId != null) { + if (caller.getType() == Account.ACCOUNT_TYPE_NORMAL) { + permittedAccounts.add(caller.getId()); + } } } From cb80ae2440aec8586cb34834cc3939d93a12beea Mon Sep 17 00:00:00 2001 From: Chiradeep Vittal Date: Fri, 7 Jun 2013 17:28:53 -0700 Subject: [PATCH 11/30] Debian repository (backports) now has latest haproxy, re-introduce into systemvm --- .../definitions/systemvmtemplate/postinstall.sh | 13 +++++++++---- .../definitions/systemvmtemplate64/postinstall.sh | 15 +++++++++++---- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/tools/appliance/definitions/systemvmtemplate/postinstall.sh b/tools/appliance/definitions/systemvmtemplate/postinstall.sh index e052cf9c1c3..203cf54d827 100644 --- a/tools/appliance/definitions/systemvmtemplate/postinstall.sh +++ b/tools/appliance/definitions/systemvmtemplate/postinstall.sh @@ -21,6 +21,12 @@ ROOTPW=password HOSTNAME=systemvm CLOUDSTACK_RELEASE=4.2.0 +add_backports () { + sed -i '/backports/d' /etc/apt/sources.list + echo 'deb http://http.us.debian.org/debian wheezy-backports main' >> /etc/apt/sources.list + apt-get update +} + install_packages() { DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical @@ -81,10 +87,7 @@ install_packages() { # rm -fr /opt/vmware-tools-distrib # apt-get -q -y --force-yes purge build-essential - # haproxy. Wheezy doesn't have haproxy, install from backports - #apt-get --no-install-recommends -q -y --force-yes install haproxy - wget http://ftp.us.debian.org/debian/pool/main/h/haproxy/haproxy_1.4.8-1_i386.deb - dpkg -i haproxy_1.4.8-1_i386.deb + apt-get --no-install-recommends -q -y --force-yes install haproxy } setup_accounts() { @@ -223,6 +226,8 @@ do_signature() { begin=$(date +%s) +echo "*************ADDING BACKPORTS********************" +add_backports echo "*************INSTALLING PACKAGES********************" install_packages echo "*************DONE INSTALLING PACKAGES********************" diff --git a/tools/appliance/definitions/systemvmtemplate64/postinstall.sh b/tools/appliance/definitions/systemvmtemplate64/postinstall.sh index 786d38d4ef0..cbcd282b62e 100644 --- a/tools/appliance/definitions/systemvmtemplate64/postinstall.sh +++ b/tools/appliance/definitions/systemvmtemplate64/postinstall.sh @@ -21,6 +21,13 @@ ROOTPW=password HOSTNAME=systemvm CLOUDSTACK_RELEASE=4.2.0 +add_backports () { + sed -i '/backports/d' /etc/apt/sources.list + echo 'deb http://http.us.debian.org/debian wheezy-backports main' >> /etc/apt/sources.list + apt-get update +} + + install_packages() { DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical @@ -80,10 +87,8 @@ install_packages() { # rm -fr /opt/vmware-tools-distrib # apt-get -q -y --force-yes purge build-essential - # haproxy. Wheezy doesn't have haproxy temporarily, install from backports - #apt-get --no-install-recommends -q -y --force-yes install haproxy - wget http://ftp.us.debian.org/debian/pool/main/h/haproxy/haproxy_1.4.8-1_amd64.deb - dpkg -i haproxy_1.4.8-1_amd64.deb + apt-get --no-install-recommends -q -y --force-yes install haproxy + } setup_accounts() { @@ -222,6 +227,8 @@ do_signature() { begin=$(date +%s) +echo "*************ADDING BACKPORTS********************" +add_backports echo "*************INSTALLING PACKAGES********************" install_packages echo "*************DONE INSTALLING PACKAGES********************" From c30d9be3cea30339cfff40c1002906634291b373 Mon Sep 17 00:00:00 2001 From: Prasanna Santhanam Date: Sat, 8 Jun 2013 13:27:08 +0530 Subject: [PATCH 12/30] Remove vm from cleanup list VM will be cleaned up when the account is cleaned up Signed-off-by: Prasanna Santhanam --- test/integration/smoke/test_routers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/integration/smoke/test_routers.py b/test/integration/smoke/test_routers.py index f6ca2790069..d89acf93736 100644 --- a/test/integration/smoke/test_routers.py +++ b/test/integration/smoke/test_routers.py @@ -107,7 +107,6 @@ class TestRouterServices(cloudstackTestCase): serviceofferingid=cls.service_offering.id ) cls.cleanup = [ - cls.vm_1, cls.account, cls.service_offering ] From 91edeb49dde411d0e0ec2ea0975c4e254b5bab91 Mon Sep 17 00:00:00 2001 From: Ron Wheeler Date: Sun, 9 Jun 2013 19:31:02 -0400 Subject: [PATCH 13/30] CLOUDSTACK-2912 - adding cd command to rpm build instructions --- docs/en-US/build-rpm.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/en-US/build-rpm.xml b/docs/en-US/build-rpm.xml index 7caf924bfe4..100a06f486e 100644 --- a/docs/en-US/build-rpm.xml +++ b/docs/en-US/build-rpm.xml @@ -48,6 +48,7 @@ under the License.
Generating RPMS Now that we have the prerequisites and source, you will cd to the packaging/centos63/ directory. + $ cd packaging/centos63 Generating RPMs is done using the package.sh script: $./package.sh From 8bc72ad55c76635939f463185291b0c283b57858 Mon Sep 17 00:00:00 2001 From: Prasanna Santhanam Date: Sat, 8 Jun 2013 20:22:16 +0530 Subject: [PATCH 14/30] Unskip all skipped tests Signed-off-by: Prasanna Santhanam --- test/integration/component/test_accounts.py | 4 ---- test/integration/component/test_egress_rules.py | 1 - test/integration/component/test_eip_elb.py | 6 ------ test/integration/component/test_netscaler_lb.py | 1 - test/integration/component/test_network_offering.py | 1 - test/integration/component/test_project_configs.py | 2 -- test/integration/component/test_project_limits.py | 1 - test/integration/component/test_projects.py | 1 - test/integration/component/test_tags.py | 2 -- test/integration/component/test_templates.py | 1 - test/integration/component/test_vm_passwdenabled.py | 1 - test/integration/component/test_vpc.py | 1 - test/integration/component/test_vpc_network.py | 4 ---- test/integration/component/test_vpc_offerings.py | 2 -- test/integration/component/test_vpc_routers.py | 4 ---- test/integration/component/test_vpc_vm_life_cycle.py | 9 --------- test/integration/component/test_vpc_vms_deployment.py | 1 - 17 files changed, 42 deletions(-) diff --git a/test/integration/component/test_accounts.py b/test/integration/component/test_accounts.py index a25e63688f3..3f106c3b048 100644 --- a/test/integration/component/test_accounts.py +++ b/test/integration/component/test_accounts.py @@ -385,7 +385,6 @@ class TestRemoveUserFromAccount(cloudstackTestCase): ) return - @unittest.skip("Open Questions") @attr(tags=["advanced", "basic", "eip", "advancedns", "sg"]) def test_02_remove_all_users(self): """Test Remove both users from the account @@ -712,7 +711,6 @@ class TestServiceOfferingSiblings(cloudstackTestCase): return -@unittest.skip("Open Questions") class TestServiceOfferingHierarchy(cloudstackTestCase): @classmethod @@ -841,7 +839,6 @@ class TestServiceOfferingHierarchy(cloudstackTestCase): return -@unittest.skip("Open Questions") class TesttemplateHierarchy(cloudstackTestCase): @classmethod @@ -1441,7 +1438,6 @@ class TestUserDetails(cloudstackTestCase): ) return -@unittest.skip("Login API response returns nothing") class TestUserLogin(cloudstackTestCase): @classmethod diff --git a/test/integration/component/test_egress_rules.py b/test/integration/component/test_egress_rules.py index 607bac86325..c8b3eeeef69 100644 --- a/test/integration/component/test_egress_rules.py +++ b/test/integration/component/test_egress_rules.py @@ -1966,7 +1966,6 @@ class TestStartStopVMWithEgressRule(cloudstackTestCase): return -@unittest.skip("Valid bug- ID: CS-12647") class TestInvalidParametersForEgress(cloudstackTestCase): def setUp(self): diff --git a/test/integration/component/test_eip_elb.py b/test/integration/component/test_eip_elb.py index 14af4a3463f..42a5148762e 100644 --- a/test/integration/component/test_eip_elb.py +++ b/test/integration/component/test_eip_elb.py @@ -182,7 +182,6 @@ class TestEIP(cloudstackTestCase): @attr(tags = ["eip"]) - @unittest.skip("skipped - Framework DB Exception") def test_01_eip_by_deploying_instance(self): """Test EIP by deploying an instance """ @@ -350,7 +349,6 @@ class TestEIP(cloudstackTestCase): return @attr(tags = ["eip"]) - @unittest.skip("skipped - Framework DB Exception") def test_02_acquire_ip_enable_static_nat(self): """Test associate new IP and enable static NAT for new IP and the VM """ @@ -495,7 +493,6 @@ class TestEIP(cloudstackTestCase): return @attr(tags = ["eip"]) - @unittest.skip("skipped - Framework DB Exception") def test_03_disable_static_nat(self): """Test disable static NAT and release EIP acquired """ @@ -695,7 +692,6 @@ class TestEIP(cloudstackTestCase): return @attr(tags = ["eip"]) - @unittest.skip("skipped - Framework DB Exception") def test_04_disable_static_nat_system(self): """Test disable static NAT with system = True """ @@ -765,7 +761,6 @@ class TestEIP(cloudstackTestCase): return @attr(tags = ["eip"]) - @unittest.skip("skipped - Framework DB Exception") def test_05_destroy_instance(self): """Test EIO after destroying instance """ @@ -1406,7 +1401,6 @@ class TestELB(cloudstackTestCase): return @attr(tags = ["eip"]) - @unittest.skip("valid bug : http://bugs.cloudstack.org/browse/CS-15077 : ListPublicIPAddress failing") def test_04_delete_lb_on_eip(self): """Test delete LB rule generated on EIP """ diff --git a/test/integration/component/test_netscaler_lb.py b/test/integration/component/test_netscaler_lb.py index 80b3f0b8b93..19888972880 100644 --- a/test/integration/component/test_netscaler_lb.py +++ b/test/integration/component/test_netscaler_lb.py @@ -2132,7 +2132,6 @@ class TestLoadBalancingRule(cloudstackTestCase): return -@unittest.skip("Questions - How to verify after changing public/private ports?") class TestDeleteCreateLBRule(cloudstackTestCase): @classmethod diff --git a/test/integration/component/test_network_offering.py b/test/integration/component/test_network_offering.py index b51d0e4c8cb..a3e87f6d61e 100644 --- a/test/integration/component/test_network_offering.py +++ b/test/integration/component/test_network_offering.py @@ -1816,7 +1816,6 @@ class TestNetworkUpgrade(cloudstackTestCase): return -@unittest.skip("Skipped since shared network requires StartIp/endIp/gateway/netmask") class TestSharedNetworkWithoutIp(cloudstackTestCase): @classmethod diff --git a/test/integration/component/test_project_configs.py b/test/integration/component/test_project_configs.py index 1eef123b2ee..641649702e1 100644 --- a/test/integration/component/test_project_configs.py +++ b/test/integration/component/test_project_configs.py @@ -343,7 +343,6 @@ class TestProjectCreationNegative(cloudstackTestCase): @attr(configuration = "allow.user.create.projects") @attr(tags = ["advanced", "basic", "sg", "eip", "advancedns", "simulator"]) - @unittest.skip("Known bug-able to create project as a domain user") def test_user_project_creation(self): """Test create project as a domain admin and domain user """ @@ -1299,7 +1298,6 @@ class TestProjectInviteTimeout(cloudstackTestCase): ) return - @unittest.skip("Requires SMPT configs") def test_09_invite_to_project_by_email(self): """Test invite user to project by email""" diff --git a/test/integration/component/test_project_limits.py b/test/integration/component/test_project_limits.py index af99717ad16..6e3f41d6465 100644 --- a/test/integration/component/test_project_limits.py +++ b/test/integration/component/test_project_limits.py @@ -328,7 +328,6 @@ class TestProjectLimits(cloudstackTestCase): return @attr(tags=["advanced", "basic", "sg", "eip", "advancedns", "simulator"]) - @unittest.skip("No provision for updating resource limits from account through API") def test_02_project_limits_normal_user(self): """ Test project limits """ diff --git a/test/integration/component/test_projects.py b/test/integration/component/test_projects.py index e1975cf5295..5f37f96804e 100644 --- a/test/integration/component/test_projects.py +++ b/test/integration/component/test_projects.py @@ -559,7 +559,6 @@ class TestDeleteAccountWithProject(cloudstackTestCase): return -@unittest.skip("Deleting domain doesn't cleanup account") class TestDeleteDomainWithProject(cloudstackTestCase): @classmethod diff --git a/test/integration/component/test_tags.py b/test/integration/component/test_tags.py index 992ca1daf58..52df6da5554 100644 --- a/test/integration/component/test_tags.py +++ b/test/integration/component/test_tags.py @@ -690,7 +690,6 @@ class TestResourceTags(cloudstackTestCase): return @attr(tags=["advanced"]) - @unittest.skip("Not supported in 3.0.5") def test_04_vpn_tag(self): """ Test Create tag on vpn and remove the vpn """ @@ -1421,7 +1420,6 @@ class TestResourceTags(cloudstackTestCase): return @attr(tags=["basic", "sg"]) - @unittest.skip("skip") def test_11_migrate_tagged_vm_del(self): """ Test migration of a tagged vm and delete the tag """ diff --git a/test/integration/component/test_templates.py b/test/integration/component/test_templates.py index 65beabfc98f..e4599d41981 100644 --- a/test/integration/component/test_templates.py +++ b/test/integration/component/test_templates.py @@ -99,7 +99,6 @@ class Services: } -@unittest.skip("Open questions") class TestCreateTemplate(cloudstackTestCase): def setUp(self): diff --git a/test/integration/component/test_vm_passwdenabled.py b/test/integration/component/test_vm_passwdenabled.py index e22a1a0a75a..ebe32c8d75c 100644 --- a/test/integration/component/test_vm_passwdenabled.py +++ b/test/integration/component/test_vm_passwdenabled.py @@ -69,7 +69,6 @@ class Services: # CentOS 5.3 (64-bit) } -@unittest.skip("Additional test") class TestVMPasswordEnabled(cloudstackTestCase): @classmethod diff --git a/test/integration/component/test_vpc.py b/test/integration/component/test_vpc.py index 7dbe7c4bf91..3fc0cc5a7e2 100644 --- a/test/integration/component/test_vpc.py +++ b/test/integration/component/test_vpc.py @@ -2474,7 +2474,6 @@ class TestVPC(cloudstackTestCase): "Updation of VPC display text failed.") -@unittest.skip("Skip") class TestVPCHostMaintenance(cloudstackTestCase): @classmethod diff --git a/test/integration/component/test_vpc_network.py b/test/integration/component/test_vpc_network.py index c0c393c4ca1..c321103a309 100644 --- a/test/integration/component/test_vpc_network.py +++ b/test/integration/component/test_vpc_network.py @@ -459,7 +459,6 @@ class TestVPCNetwork(cloudstackTestCase): ) return - @unittest.skip("Skip - Requires netscaler setup") @attr(tags=["netscaler", "intervlan"]) def test_03_create_network_netscaler(self): """ Test create network using netscaler for LB @@ -707,7 +706,6 @@ class TestVPCNetwork(cloudstackTestCase): return @attr(tags=["advanced", "intervlan"]) - @unittest.skip("Skipping - able to create network with RvR") def test_06_create_network_with_rvr(self): """ Test create network with eredundant router capability """ @@ -1038,7 +1036,6 @@ class TestVPCNetwork(cloudstackTestCase): "Network creation failed as VPC support nw with conserve mode OFF") return -@unittest.skip("tested") class TestVPCNetworkRanges(cloudstackTestCase): @classmethod @@ -1670,7 +1667,6 @@ class TestVPCNetworkUpgrade(cloudstackTestCase): return @attr(tags=["advanced", "intervlan"]) - @unittest.skip("Error while NW upgrade - Failed to implement network (with specified id) elements and resources as a part of network update") def test_01_network_services_upgrade(self): """ Test update Network that is part of a VPC to a network offering that has more services. diff --git a/test/integration/component/test_vpc_offerings.py b/test/integration/component/test_vpc_offerings.py index 4b9877506f2..a111d7d9059 100644 --- a/test/integration/component/test_vpc_offerings.py +++ b/test/integration/component/test_vpc_offerings.py @@ -278,7 +278,6 @@ class TestVPCOffering(cloudstackTestCase): return @attr(tags=["advanced", "intervlan"]) - @unittest.skip("Skipping - Issue: Deleting account doesn't clean VPC") def test_02_deploy_vms_in_vpc_nw(self): """Test deploy virtual machines in VPC networks""" @@ -897,7 +896,6 @@ class TestVPCOffering(cloudstackTestCase): return @attr(tags=["advanced", "intervlan"]) - @unittest.skip("Skipping - API should not allow to create VPC offering without SourceNAT, Firewall") def test_06_vpc_off_invalid_services(self): """Test VPC offering with invalid services""" diff --git a/test/integration/component/test_vpc_routers.py b/test/integration/component/test_vpc_routers.py index 7dc95e826e3..a8559e5cc6c 100644 --- a/test/integration/component/test_vpc_routers.py +++ b/test/integration/component/test_vpc_routers.py @@ -509,7 +509,6 @@ class TestVPCRoutersBasic(cloudstackTestCase): ) return - @unittest.skip("Needs hosts") @attr(tags=["advanced", "intervlan"]) def test_04_migrate_router_after_creating_vpc(self): """ Test migration of router to another host after creating VPC """ @@ -531,7 +530,6 @@ class TestVPCRoutersBasic(cloudstackTestCase): self.migrate_router(routers[0]) return - @unittest.skip("Fails") @attr(tags=["advanced", "intervlan"]) def test_05_change_service_offerring_vpc(self): """ Tests to change service offering of the Router after @@ -1236,7 +1234,6 @@ class TestVPCRouterOneNetwork(cloudstackTestCase): ) return - @unittest.skip("Untested - hosts not available") @attr(tags=["advanced", "intervlan"]) def test_04_migrate_router_after_addition_of_one_guest_network(self): """ Test migrate of router after addition of one guest network @@ -1280,7 +1277,6 @@ class TestVPCRouterOneNetwork(cloudstackTestCase): self.migrate_router(routers[0]) return - @unittest.skip("Fails") @attr(tags=["advanced", "intervlan"]) def test_05_chg_srv_off_router_after_addition_of_one_guest_network(self): """ Test to change service offering of router after addition of one guest network diff --git a/test/integration/component/test_vpc_vm_life_cycle.py b/test/integration/component/test_vpc_vm_life_cycle.py index 7658bebe2e2..44f72880f39 100644 --- a/test/integration/component/test_vpc_vm_life_cycle.py +++ b/test/integration/component/test_vpc_vm_life_cycle.py @@ -2464,7 +2464,6 @@ class TestVMLifeCycleStoppedVPCVR(cloudstackTestCase): ) return - @unittest.skip("Skipping - Not able to SSH if VPCVR is stopped") @attr(tags=["advanced", "intervlan"]) def test_03_start_instance_in_network(self): """ Test start an instance in VPC networks @@ -2488,7 +2487,6 @@ class TestVMLifeCycleStoppedVPCVR(cloudstackTestCase): self.validate_network_rules() return - @unittest.skip("Skipping - Not able to SSH if VPCVR is stopped") @attr(tags=["advanced", "intervlan"]) def test_04_reboot_instance_in_network(self): """ Test reboot an instance in VPC networks @@ -2516,7 +2514,6 @@ class TestVMLifeCycleStoppedVPCVR(cloudstackTestCase): self.validate_network_rules() return - @unittest.skip("Skipping - Not able to SSH if VPCVR is stopped") @attr(tags=["advanced", "intervlan"]) def test_05_destroy_instance_in_network(self): """ Test destroy an instance in VPC networks @@ -2562,7 +2559,6 @@ class TestVMLifeCycleStoppedVPCVR(cloudstackTestCase): ) return - @unittest.skip("Skipping - Not able to SSH if VPCVR is stopped") @attr(tags=["advanced", "intervlan"]) def test_06_recover_instance_in_network(self): """ Test recover an instance in VPC networks @@ -2594,7 +2590,6 @@ class TestVMLifeCycleStoppedVPCVR(cloudstackTestCase): self.validate_network_rules() return - @unittest.skip("Skipping - Not able to SSH if VPCVR is stopped") @attr(tags=["advanced", "intervlan"]) def test_07_migrate_instance_in_network(self): """ Test migrate an instance in VPC networks @@ -2645,7 +2640,6 @@ class TestVMLifeCycleStoppedVPCVR(cloudstackTestCase): self.validate_network_rules() return - @unittest.skip("Skipping - Not able to SSH if VPCVR is stopped") @attr(tags=["advanced", "intervlan"]) def test_08_user_data(self): """ Test user data in virtual machines @@ -2700,7 +2694,6 @@ class TestVMLifeCycleStoppedVPCVR(cloudstackTestCase): ) return - @unittest.skip("Skipping - Not able to SSH if VPCVR is stopped") @attr(tags=["advanced", "intervlan"]) def test_09_meta_data(self): """ Test meta data in virtual machines @@ -2754,7 +2747,6 @@ class TestVMLifeCycleStoppedVPCVR(cloudstackTestCase): ) return - @unittest.skip("Skipping - Not able to SSH if VPCVR is stopped") @attr(tags=["advanced", "intervlan"]) def test_10_expunge_instance_in_network(self): """ Test expunge an instance in VPC networks @@ -2812,7 +2804,6 @@ class TestVMLifeCycleStoppedVPCVR(cloudstackTestCase): return -@unittest.skip("Skip - Requires Tagged hosts setup") class TestVMLifeCycleDiffHosts(cloudstackTestCase): @classmethod diff --git a/test/integration/component/test_vpc_vms_deployment.py b/test/integration/component/test_vpc_vms_deployment.py index c49ef458a40..c599d96113e 100644 --- a/test/integration/component/test_vpc_vms_deployment.py +++ b/test/integration/component/test_vpc_vms_deployment.py @@ -1721,7 +1721,6 @@ class TestVMDeployVPC(cloudstackTestCase): return @attr(tags=["advanced", "intervlan"]) - @unittest.skip("Not tested") def test_07_delete_network_with_rules(self): """ Test delete network that has PF/staticNat/LB rules/Network Acl """ From 22a850828001b89fd6ac8e3916256bc7e966a1ef Mon Sep 17 00:00:00 2001 From: Likitha Shetty Date: Fri, 7 Jun 2013 19:22:39 +0530 Subject: [PATCH 15/30] CLOUDSTACK-2894. Removing all vlan ranges should update the vnet column to NULL. --- .../com/cloud/network/NetworkServiceImpl.java | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/server/src/com/cloud/network/NetworkServiceImpl.java b/server/src/com/cloud/network/NetworkServiceImpl.java index c2af8e8fe60..c7be2c62cf1 100755 --- a/server/src/com/cloud/network/NetworkServiceImpl.java +++ b/server/src/com/cloud/network/NetworkServiceImpl.java @@ -2680,9 +2680,10 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService { vnetString = vnetString+vnetRange.first().toString()+"-"+value.toString()+";"; } } - vnetString = vnetString+"*"; - vnetString = vnetString.replace(";*",""); - network.setVnet(vnetString); + if (vnetString.length() > 0 && vnetString.charAt(vnetString.length()-1)==';') { + vnetString = vnetString.substring(0, vnetString.length()-1); + } + network.setVnet(vnetString); } for (Pair vnetToAdd : vnetsToAdd) { @@ -2788,12 +2789,15 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService { _datacneter_vnet.deleteRange(txn, network.getDataCenterId(), network.getId(), start, end); String vnetString=""; - for (Pair vnetRange : existingRanges ){ - vnetString=vnetString+vnetRange.first().toString()+"-"+vnetRange.second().toString()+";"; + if (existingRanges.isEmpty()) { + network.setVnet(null); + } else { + for (Pair vnetRange : existingRanges ) { + vnetString=vnetString+vnetRange.first().toString()+"-"+vnetRange.second().toString()+";"; + } + vnetString = vnetString.substring(0, vnetString.length()-1); + network.setVnet(vnetString); } - vnetString = vnetString+"*"; - vnetString = vnetString.replace(";*",""); - network.setVnet(vnetString); _physicalNetworkDao.update(network.getId(), network); txn.commit(); _physicalNetworkDao.releaseFromLockTable(network.getId()); From 673b293d75419a4b76379092db2b204cdc6160a4 Mon Sep 17 00:00:00 2001 From: Jayapal Date: Wed, 5 Jun 2013 19:13:03 +0530 Subject: [PATCH 16/30] CLOUDSTACK-2761 Fixed PF/StaticNAT in vmware vpc Signed-off-by: Abhinandan Prateek --- .../vmware/resource/VmwareResource.java | 48 ++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java index 5944cc864f1..46219c38d5f 100755 --- a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -811,7 +811,53 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa return new SetFirewallRulesAnswer(cmd, true, results); } + protected SetStaticNatRulesAnswer SetVPCStaticNatRules(SetStaticNatRulesCommand cmd) { + if (s_logger.isInfoEnabled()) { + s_logger.info("Executing resource SetVPCStaticNatRulesCommand: " + _gson.toJson(cmd)); + } + + String[] results = new String[cmd.getRules().length]; + VmwareManager mgr = getServiceContext().getStockObject(VmwareManager.CONTEXT_STOCK_NAME); + String controlIp = getRouterSshControlIp(cmd); + + int i = 0; + boolean endResult = true; + for (StaticNatRuleTO rule : cmd.getRules()) { + // Prepare command to be send to VPC VR + String args = ""; + args += rule.revoked() ? " -D" : " -A"; + args += " -l " + rule.getSrcIp(); + args += " -r " + rule.getDstIp(); + + // Invoke command on VPC VR. + try { + Pair result = SshHelper.sshExecute(controlIp, DEFAULT_DOMR_SSHPORT, "root", mgr.getSystemVMKeyFile(), null, "/opt/cloud/bin/vpc_staticnat.sh " + args); + + if (s_logger.isDebugEnabled()) + s_logger.debug("Executing script on domain router " + controlIp + ": /opt/cloud/bin/vpc_staticnat.sh " + args); + + if (!result.first()) { + s_logger.error("SetVPCStaticNatRulesCommand failure on setting one rule. args: " + args); + results[i++] = "Failed"; + endResult = false; + } else { + results[i++] = null; + } + } catch (Throwable e) { + s_logger.error("SetVPCStaticNatRulesCommand (args: " + args + ") failed on setting one rule due to " + VmwareHelper.getExceptionMessage(e), e); + results[i++] = "Failed"; + endResult = false; + } + } + return new SetStaticNatRulesAnswer(cmd, results, endResult); + } + protected Answer execute(SetStaticNatRulesCommand cmd) { + + if (cmd.getVpcId() != null) { + return SetVPCStaticNatRules(cmd); + } + if (s_logger.isInfoEnabled()) { s_logger.info("Executing resource SetFirewallRuleCommand: " + _gson.toJson(cmd)); } @@ -1262,7 +1308,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa args += rule.revoked() ? " -D" : " -A"; args += " -P " + rule.getProtocol().toLowerCase(); args += " -l " + rule.getSrcIp(); - args += " -p " + rule.getStringSrcPortRange().replace(":", "-"); + args += " -p " + rule.getStringSrcPortRange(); args += " -r " + rule.getDstIp(); args += " -d " + rule.getStringDstPortRange().replace(":", "-"); From c1ad3b7974449f457a1cc4e50fe7af260d1c5bf6 Mon Sep 17 00:00:00 2001 From: Jayapal Date: Thu, 23 May 2013 16:10:44 +0530 Subject: [PATCH 17/30] CLOUDSTACK-2604 Fixed deleting secondary ip when no PF rules set Signed-off-by: Abhinandan Prateek --- api/src/com/cloud/network/NetworkModel.java | 2 ++ .../network/rules/dao/PortForwardingRulesDao.java | 2 +- .../rules/dao/PortForwardingRulesDaoImpl.java | 9 ++++++++- server/src/com/cloud/network/NetworkModelImpl.java | 5 +++-- .../src/com/cloud/network/NetworkServiceImpl.java | 13 +++++++++---- .../com/cloud/network/rules/RulesManagerImpl.java | 2 +- .../com/cloud/network/MockNetworkModelImpl.java | 5 +++++ server/test/com/cloud/vpc/MockNetworkModelImpl.java | 5 +++++ 8 files changed, 34 insertions(+), 9 deletions(-) diff --git a/api/src/com/cloud/network/NetworkModel.java b/api/src/com/cloud/network/NetworkModel.java index f84a8b0c76a..05307eb47aa 100644 --- a/api/src/com/cloud/network/NetworkModel.java +++ b/api/src/com/cloud/network/NetworkModel.java @@ -272,4 +272,6 @@ public interface NetworkModel { Map getNtwkOffDetails(long offId); Networks.IsolationType[] listNetworkIsolationMethods(); + + Nic getNicInNetworkIncludingRemoved(long vmId, long networkId); } \ No newline at end of file diff --git a/engine/schema/src/com/cloud/network/rules/dao/PortForwardingRulesDao.java b/engine/schema/src/com/cloud/network/rules/dao/PortForwardingRulesDao.java index 682a941856c..9a1d321ae91 100644 --- a/engine/schema/src/com/cloud/network/rules/dao/PortForwardingRulesDao.java +++ b/engine/schema/src/com/cloud/network/rules/dao/PortForwardingRulesDao.java @@ -43,5 +43,5 @@ public interface PortForwardingRulesDao extends GenericDao listByAccount(long accountId); List listByDestIpAddr(String ip4Address); - + PortForwardingRuleVO findByIdAndIp(long id, String secondaryIp); } diff --git a/engine/schema/src/com/cloud/network/rules/dao/PortForwardingRulesDaoImpl.java b/engine/schema/src/com/cloud/network/rules/dao/PortForwardingRulesDaoImpl.java index cc780cbd446..c0db7800378 100644 --- a/engine/schema/src/com/cloud/network/rules/dao/PortForwardingRulesDaoImpl.java +++ b/engine/schema/src/com/cloud/network/rules/dao/PortForwardingRulesDaoImpl.java @@ -158,5 +158,12 @@ public class PortForwardingRulesDaoImpl extends GenericDaoBase sc = AllFieldsSearch.create(); + sc.setParameters("id", id); + sc.setParameters("dstIp", secondaryIp); + return findOneBy(sc); + } } diff --git a/server/src/com/cloud/network/NetworkModelImpl.java b/server/src/com/cloud/network/NetworkModelImpl.java index 21917f76351..fa34d65ab1a 100755 --- a/server/src/com/cloud/network/NetworkModelImpl.java +++ b/server/src/com/cloud/network/NetworkModelImpl.java @@ -770,7 +770,7 @@ public class NetworkModelImpl extends ManagerBase implements NetworkModel { @Override public Nic getNicInNetwork(long vmId, long networkId) { - return _nicDao.findByInstanceIdAndNetworkIdIncludingRemoved(networkId, vmId); + return _nicDao.findByNtwkIdAndInstanceId(networkId, vmId); } @Override @@ -1761,7 +1761,8 @@ public class NetworkModelImpl extends ManagerBase implements NetworkModel { return true; } - Nic getNicInNetworkIncludingRemoved(long vmId, long networkId) { + @Override + public Nic getNicInNetworkIncludingRemoved(long vmId, long networkId) { return _nicDao.findByInstanceIdAndNetworkIdIncludingRemoved(networkId, vmId); } diff --git a/server/src/com/cloud/network/NetworkServiceImpl.java b/server/src/com/cloud/network/NetworkServiceImpl.java index c7be2c62cf1..d5a59d62817 100755 --- a/server/src/com/cloud/network/NetworkServiceImpl.java +++ b/server/src/com/cloud/network/NetworkServiceImpl.java @@ -789,10 +789,15 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService { if (dc.getNetworkType() == NetworkType.Advanced && network.getGuestType() == Network.GuestType.Isolated) { //check PF or static NAT is configured on this ip address String secondaryIp = secIpVO.getIp4Address(); - List pfRuleList = _portForwardingDao.listByDestIpAddr(secondaryIp); - if (pfRuleList.size() != 0) { - s_logger.debug("VM nic IP " + secondaryIp + " is associated with the port forwarding rule"); - throw new InvalidParameterValueException("Can't remove the secondary ip " + secondaryIp + " is associate with the port forwarding rule"); + List fwRulesList = _firewallDao.listByNetworkAndPurpose(network.getId(), Purpose.PortForwarding); + + if (fwRulesList.size() != 0) { + for (FirewallRuleVO rule: fwRulesList) { + if (_portForwardingDao.findByIdAndIp(rule.getId(), secondaryIp) != null) { + s_logger.debug("VM nic IP " + secondaryIp + " is associated with the port forwarding rule"); + throw new InvalidParameterValueException("Can't remove the secondary ip " + secondaryIp + " is associate with the port forwarding rule"); + } + } } //check if the secondary ip associated with any static nat rule IPAddressVO publicIpVO = _ipAddressDao.findByVmIp(secondaryIp); diff --git a/server/src/com/cloud/network/rules/RulesManagerImpl.java b/server/src/com/cloud/network/rules/RulesManagerImpl.java index 41bf2b3af65..bcda32d1d64 100755 --- a/server/src/com/cloud/network/rules/RulesManagerImpl.java +++ b/server/src/com/cloud/network/rules/RulesManagerImpl.java @@ -1371,7 +1371,7 @@ public class RulesManagerImpl extends ManagerBase implements RulesManager, Rules // create new static nat rule // Get nic IP4 address - Nic guestNic = _networkModel.getNicInNetwork(vm.getId(), networkId); + Nic guestNic = _networkModel.getNicInNetworkIncludingRemoved(vm.getId(), networkId); if (guestNic == null) { throw new InvalidParameterValueException("Vm doesn't belong to the network with specified id"); } diff --git a/server/test/com/cloud/network/MockNetworkModelImpl.java b/server/test/com/cloud/network/MockNetworkModelImpl.java index c3a0d6c5ae9..3a367480a30 100644 --- a/server/test/com/cloud/network/MockNetworkModelImpl.java +++ b/server/test/com/cloud/network/MockNetworkModelImpl.java @@ -874,4 +874,9 @@ public class MockNetworkModelImpl extends ManagerBase implements NetworkModel { // TODO Auto-generated method stub return null; } + + @Override + public Nic getNicInNetworkIncludingRemoved(long vmId, long networkId) { + return null; + } } diff --git a/server/test/com/cloud/vpc/MockNetworkModelImpl.java b/server/test/com/cloud/vpc/MockNetworkModelImpl.java index d9e33b75616..3e67f5e76e2 100644 --- a/server/test/com/cloud/vpc/MockNetworkModelImpl.java +++ b/server/test/com/cloud/vpc/MockNetworkModelImpl.java @@ -887,4 +887,9 @@ public class MockNetworkModelImpl extends ManagerBase implements NetworkModel { return null; } + @Override + public Nic getNicInNetworkIncludingRemoved(long vmId, long networkId) { + return null; + } + } From c8d607eae5db696b20f797a3297ad5eb5c746115 Mon Sep 17 00:00:00 2001 From: Jayapal Date: Fri, 24 May 2013 11:52:25 +0530 Subject: [PATCH 18/30] CLOUDSTACK-2650 setting vm ip null in user_ip_address when static nat ip Signed-off-by: Abhinandan Prateek --- engine/schema/src/com/cloud/network/dao/IPAddressDaoImpl.java | 1 + 1 file changed, 1 insertion(+) diff --git a/engine/schema/src/com/cloud/network/dao/IPAddressDaoImpl.java b/engine/schema/src/com/cloud/network/dao/IPAddressDaoImpl.java index 1839ca45476..886011ecc31 100755 --- a/engine/schema/src/com/cloud/network/dao/IPAddressDaoImpl.java +++ b/engine/schema/src/com/cloud/network/dao/IPAddressDaoImpl.java @@ -155,6 +155,7 @@ public class IPAddressDaoImpl extends GenericDaoBase implemen address.setAssociatedWithNetworkId(null); address.setVpcId(null); address.setSystem(false); + address.setVmIp(null); update(ipAddressId, address); } From 6e2454228313c3373bfa96ef623a4b6ceb88d0ee Mon Sep 17 00:00:00 2001 From: kyrameng Date: Sun, 9 Jun 2013 19:09:20 -0400 Subject: [PATCH 19/30] meng's proposal for GSOC2013 --- docs/en-US/CloudStack_GSoC_Guide.xml | 2 +- docs/en-US/gsoc-meng.xml | 235 +++++++++++++++++++++++++++ 2 files changed, 236 insertions(+), 1 deletion(-) create mode 100644 docs/en-US/gsoc-meng.xml diff --git a/docs/en-US/CloudStack_GSoC_Guide.xml b/docs/en-US/CloudStack_GSoC_Guide.xml index 243a0ca361b..1f435931363 100644 --- a/docs/en-US/CloudStack_GSoC_Guide.xml +++ b/docs/en-US/CloudStack_GSoC_Guide.xml @@ -49,6 +49,6 @@ - + diff --git a/docs/en-US/gsoc-meng.xml b/docs/en-US/gsoc-meng.xml new file mode 100644 index 00000000000..1de259dcac1 --- /dev/null +++ b/docs/en-US/gsoc-meng.xml @@ -0,0 +1,235 @@ + + +%BOOK_ENTITIES; +]> + + + + + Meng's 2013 GSoC Proposal + This chapter describes Meng's 2013 Google Summer of Code project within the &PRODUCT; ASF project. It is a copy paste of the submitted proposal. +
+ Project Description + + Getting a hadoop cluster going can be challenging and painful due to the tedious configuration phase and the diverse idiosyncrasies of each cloud provider. Apache Whirr[1] and Provisionr is a set of libraries for running cloud services in an automatic or semi-automatic fashion. They take advantage of a cloud-neutral library called jclouds[2] to create one-click, auto-configuring hadoop clusters on multiple clouds. Since jclouds supports CloudStack API, most of the services provided by Whirr and Provisionr should work out of the box on CloudStack. My first task is to test that assumption, make sure everything is well documented, and correct all issues with the latest version of CloudStack (4.0 and 4.1). + + + +The biggest challenge for hadoop provisioning is automatically configuring each instance at launch time based on what it is supposed to do, a process known as contextualization[3][4]. It causes last minute changes inside an instance to adapt to a cluster environment. Many automated cloud services are enabled by contextualization. For example in one-click hadoop clusters, contextualization basically amounts to generating and distributing ssh key pairs among instances, telling an instance where the master node is and what other slave nodes it should be aware of, etc. On EC2 contextualization is done via passing information through the EC2_USER_DATA entry[5][6]. Whirr and Provisionr embrace this feature to provision hadoop instances on EC2. My second task is to test and extend Whirr and Provisionr’s one-click solution on EC2 to CloudStack and also improve CloudStack’s support for Whirr and Provisionr to enable hadoop provisioning on CloudStack based clouds. + + +My third task is to add a Query API that is compatible with Amazon Elastic MapReduce (EMR) to CloudStack. Through this API, all hadoop provisioning functionality will be exposed and users can reuse cloud clients that are written for EMR to create and manage hadoop clusters on CloudStack based clouds. + +
+ +
+ Project Details + + Whirr defines four roles for the hadoop provisioning service: Namenode, JobTracker, Datanode and TaskTraker. With the help of CloudInit[7] (a popular package for cloud instance initialization), each VM instance is configured based on its role and a compressed file that is passed in the EC2_USER_DATA entry. Since CloudStack also supports EC2_USER_DATA, I think the most feasible way to have hadoop provisioning on CloudStack is to extend Whirr’s solution on EC2 to CloudStack platform and to make necessary adjustment based on CloudStack’s + + + + Whirr and Provisionr deal with two critical issues in their role configuration scripts (configure-hadoop-role_list): SSH key authentication and hostname configuration. + + + + SSH Key Authentication. The need for SSH Key based authentication is required so that the master node can login to slave nodes to start/stop hadoop daemons. Also each node needs to login to itself to start its own hadoop daemons. Traditionally this is done by generating a key pair on the master node and distributing the public key to all slave nodes. This can be only done with human intervention. Whirr works around this problem on EC2 by having a common key pair for all nodes in a hadoop cluster. Thus every node is able to login to one another. The key pair is provided by users and obtained by CloudInit inside an instance from metadata service. As far as I know, Cloudstack does not support user-provided ssh key authentication. Although CloudStack has the createSSHKeyPair API[8] to generate SSH keys and users can create an instance template that supports SSH keys, there is no easy way to have a unified SSH key on all cluster instances. Besides Whirr prefers minimal image management, so having a customized template doesn’t seem quite fit here. + + + Hostname configuration. The hostname of each instance has to be properly set and injected into the set of hadoop config files (core-site.xml, hdfs-site.xml, mapred-site.xml ). For an EC2 instance, its host name is converted from a combination of its public IP and an EC2-specific pre/suffix (e.g. an instance with IP 54.224.206.71 will have its hostname set to ec2-54-224-206-71.compute-1.amazonaws.com). This hostname amounts to the Fully Qualified Domain Name that uniquely identifies this node on the network. As for the case of CloudStack, if users do not specify a name the hostname that identifies a VM on a network will be a unique UUID generated by CloudStack[9]. + + + + + + + These two are the main issues that need support improvement on the CloudStack side. Other things like preparing disks, installing hadoop tarballs and starting hadoop daemons can be easily done as they are relatively role/instance-independent and static. Runurl can be used to simplify user-data scripts. + + + + + + After we achieve hadoop provisioning on CloudStack using Whirr we can go further to add a Query API to CloudStack to expose this functionality. I will write an API that is compatible with Amazon Elastic MapReduce Service (EMR)[10] so that users can reuse clients that are written for EMR to submit jobs to existing hadoop clusters, poll job status, terminate a hadoop instance and do other things on CloudStack based clouds. There are eight actions[11] now supported in EMR API. I will try to implement as many as I can during the period of GSoC. The following statements give some examples of the API that I will write. + + + +This will launch a new hadoop cluster with four instances using specified instance types and add a job flow to it. + + + +This will add a step to the existing job flow with ID j-3UN6WX5RRO2AG. This step will run the specified jar file. + + + +This will return the status of the given job flow. + +
+ +
+ Roadmap + + Jun. 17 ∼ Jun. 30 + + + Learn CloudStack and Apache Whirr/Provisionr APIs; Deploy a CloudStack cluster. + + + + Identify how EC2_USER_DATA is passed and executed on each CloudStack instance. + + + Figure out how the files passed in EC2_USER_DATA are acted upon by CloudInit. + + + Identify files in /etc/init/ that are used or modified by Whirr and Provisionr for hadoop related configuration. + + + Deploy a hadoop cluster on CloudStack via Whirr/Provisionr. This is to test what are missing in CloudStack or Whirr/Provisionr in terms of their support for each other. + + + Jul. 1∼ Aug. 1 + + + Write scripts to configure VM hostname on CloudStack with the help of CloudInit; + + + Write scripts to distribute SSH keys among CloudStack instances. Add the capability of using user-provided ssh key for authentication to CloudStack. + + + Take care of the other things left for hadoop provisioning, such as mounting disks, installing hadoop tarballs, etc. + + + Compose files that need to be passed in EC2_USER_DATA to each CloudStack instance . Test these files and write patches to make sure that Whirr/Provisionr can succefully deploy one-click hadoop clusters on CloudStack. + + + Aug. 3 ∼ Sep. 8 + + + Design and build an Elastic Mapreduce API for CloudStack that takes control of hadoop cluster creation and management. + + + Implement the eight actions defined in EMR API. This task might take a while. + + + + Sep. 10 ∼ Sep. 23 + + + + Code cleaning and documentation wrap up. + + + + + + +
+ +
+ Deliverables + + + + Whirr has limited support for CloudStack. Check what’s missing and make sure all steps are properly documented on the Whirr and CloudStack websites. + + + Contribute code to CloudStack and and send patches to Whirr/Provisionr if necessary to enable hadoop provisioning on CloudStack via Whirr/Provisionr. + + + Build an EMR-compatible API for CloudStack. + + +
+
+ Nice to have + In addition to the required deliverables, it’s nice to have the following: + + + + The capability to add and remove hadoop nodes dynamically to enable elastic hadoop clusters on CloudStack. + + + + A review of the existing tools that offer one-click provisioning and make sure that they support CloudStack based clouds. + + +
+ +
+ References + + + + + http://whirr.apache.org/ + + + http://www.jclouds.org/documentation/gettingstarted/what-is-jclouds/ + + + Katarzyna Keahey, Tim Freeman, Contextualization: Providing One-Click Virtual Clusters + + + http://www.nimbusproject.org/docs/current/clouds/clusters2.html + + + http://aws.amazon.com/amazon-linux-ami/ + + + https://svn.apache.org/repos/asf/whirr/branches/contrib-python/src/py/hadoop/cloud/data/hadoop-ec2-init-remote.sh + + + https://help.ubuntu.com/community/CloudInit + + + http://cloudstack.apache.org/docs/en-US/Apache_CloudStack/4.0.2/html/Installation_Guide/using-sshkeys.html + + + https://cwiki.apache.org/CLOUDSTACK/allow-user-provided-hostname-internal-vm-name-on-hypervisor-instead-of-cloud-platform-auto-generated-name-for-guest-vms.html + + +http://docs.aws.amazon.com/ElasticMapReduce/latest/API/Welcome.html + + + http://docs.aws.amazon.com/ElasticMapReduce/latest/API/API_Operations.html + + + http://buildacloud.org/blog/235-puppet-and-cloudstack.html + + +http://chriskleban-internet.blogspot.com/2012/03/build-cloud-cloudstack-instance.html + + + http://gehrcke.de/2009/06/aws-about-api/ + + + Apache_CloudStack-4.0.0-incubating-API_Developers_Guide-en-US.pdf + + + +
+ + From 52f9d5efe238c5fe7a36c2d7094f3f73ca9357cd Mon Sep 17 00:00:00 2001 From: Prasanna Santhanam Date: Mon, 10 Jun 2013 13:36:13 +0530 Subject: [PATCH 20/30] Improve debug message when failing the test Added the listVM response as part of debug logging Signed-off-by: Prasanna Santhanam --- test/integration/smoke/test_vm_life_cycle.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/integration/smoke/test_vm_life_cycle.py b/test/integration/smoke/test_vm_life_cycle.py index afe9b8a6331..9aaa13fcb57 100644 --- a/test/integration/smoke/test_vm_life_cycle.py +++ b/test/integration/smoke/test_vm_life_cycle.py @@ -687,10 +687,12 @@ class TestVMLifeCycle(cloudstackTestCase): else: break + self.debug("listVirtualMachines response: %s" % list_vm_response) + self.assertEqual( list_vm_response, None, - "Check Expunged virtual machine is listVirtualMachines" + "Check Expunged virtual machine is in listVirtualMachines response" ) return From 20c1f2c3466926163ccf0fe1487b3ca61d35da07 Mon Sep 17 00:00:00 2001 From: Prasanna Santhanam Date: Mon, 10 Jun 2013 13:36:38 +0530 Subject: [PATCH 21/30] CLOUDSTACK-2914: lbScheme Public should be specified in NetworkOffering NetworkOfferings now require a mandatory scheme in the serviceCapabilityList to create a VPC loadbalancer on the public side. This commit fixes the test for VPC networks. Additionally there needs to be a fix for making this the default behaviour so as not to hurt the backwards compatibility. test still fails because of CLOUDSTACK-2915 however which is a related network ACL backwards compat issue. See bug for more details. Signed-off-by: Prasanna Santhanam --- .../integration/component/test_vpc_network.py | 271 +++++++++--------- 1 file changed, 138 insertions(+), 133 deletions(-) diff --git a/test/integration/component/test_vpc_network.py b/test/integration/component/test_vpc_network.py index c321103a309..a997f43a612 100644 --- a/test/integration/component/test_vpc_network.py +++ b/test/integration/component/test_vpc_network.py @@ -35,139 +35,144 @@ class Services: def __init__(self): self.services = { - "account": { - "email": "test@test.com", - "firstname": "Test", - "lastname": "User", - "username": "test", - # Random characters are appended for unique - # username - "password": "password", - }, - "service_offering": { - "name": "Tiny Instance", - "displaytext": "Tiny Instance", - "cpunumber": 1, - "cpuspeed": 100, - "memory": 128, - }, - "network_offering": { - "name": 'VPC Network offering', - "displaytext": 'VPC Network off', - "guestiptype": 'Isolated', - "supportedservices": 'Vpn,Dhcp,Dns,SourceNat,PortForwarding,Lb,UserData,StaticNat,NetworkACL', - "traffictype": 'GUEST', - "availability": 'Optional', - "useVpc": 'on', - "serviceProviderList": { - "Vpn": 'VpcVirtualRouter', - "Dhcp": 'VpcVirtualRouter', - "Dns": 'VpcVirtualRouter', - "SourceNat": 'VpcVirtualRouter', - "PortForwarding": 'VpcVirtualRouter', - "Lb": 'VpcVirtualRouter', - "UserData": 'VpcVirtualRouter', - "StaticNat": 'VpcVirtualRouter', - "NetworkACL": 'VpcVirtualRouter' - }, - "servicecapabilitylist": { - }, - }, - "network_off_netscaler": { - "name": 'Network offering-netscaler', - "displaytext": 'Network offering-netscaler', - "guestiptype": 'Isolated', - "supportedservices": 'Dhcp,Dns,SourceNat,PortForwarding,Vpn,Lb,UserData,StaticNat', - "traffictype": 'GUEST', - "availability": 'Optional', - "useVpc": 'on', - "serviceProviderList": { - "Dhcp": 'VpcVirtualRouter', - "Dns": 'VpcVirtualRouter', - "SourceNat": 'VpcVirtualRouter', - "PortForwarding": 'VpcVirtualRouter', - "Vpn": 'VpcVirtualRouter', - "Lb": 'Netscaler', - "UserData": 'VpcVirtualRouter', - "StaticNat": 'VpcVirtualRouter', - }, - }, - "network_off_shared": { - "name": 'Shared Network offering', - "displaytext": 'Shared Network offering', - "guestiptype": 'Shared', - "traffictype": 'GUEST', - "availability": 'Optional', - "useVpc": 'on', - "specifyIpRanges": True, - "specifyVlan": True - }, - "vpc_offering": { - "name": 'VPC off', - "displaytext": 'VPC off', - "supportedservices": 'Dhcp,Dns,SourceNat,PortForwarding,Vpn,Lb,UserData,StaticNat', - }, - "vpc": { - "name": "TestVPC", - "displaytext": "TestVPC", - "cidr": '10.0.0.1/24' - }, - "network": { - "name": "Test Network", - "displaytext": "Test Network", - "netmask": '255.255.255.0' - }, - "lbrule": { - "name": "SSH", - "alg": "leastconn", - # Algorithm used for load balancing - "privateport": 22, - "publicport": 2222, - "openfirewall": False, - "startport": 22, - "endport": 2222, - "protocol": "TCP", - "cidrlist": '0.0.0.0/0', - }, - "natrule": { - "privateport": 22, - "publicport": 22, - "startport": 22, - "endport": 22, - "protocol": "TCP", - "cidrlist": '0.0.0.0/0', - }, - "fw_rule": { - "startport": 1, - "endport": 6000, - "cidr": '0.0.0.0/0', - # Any network (For creating FW rule) - "protocol": "TCP" - }, - "http_rule": { - "startport": 80, - "endport": 80, - "cidrlist": '0.0.0.0/0', - "protocol": "ICMP" - }, - "virtual_machine": { - "displayname": "Test VM", - "username": "root", - "password": "password", - "ssh_port": 22, - "hypervisor": 'XenServer', - # Hypervisor type should be same as - # hypervisor type of cluster - "privateport": 22, - "publicport": 22, - "protocol": 'TCP', - }, - "ostype": 'CentOS 5.3 (64-bit)', - # Cent OS 5.3 (64 bit) - "sleep": 60, - "timeout": 10, - "mode": 'advanced' - } + "account": { + "email": "test@test.com", + "firstname": "Test", + "lastname": "User", + "username": "test", + # Random characters are appended for unique + # username + "password": "password", + }, + "service_offering": { + "name": "Tiny Instance", + "displaytext": "Tiny Instance", + "cpunumber": 1, + "cpuspeed": 100, + "memory": 128, + }, + "network_offering": { + "name": 'VPC Network offering', + "displaytext": 'VPC Network off', + "guestiptype": 'Isolated', + "supportedservices": 'Vpn,Dhcp,Dns,SourceNat,PortForwarding,Lb,UserData,StaticNat,NetworkACL', + "traffictype": 'GUEST', + "availability": 'Optional', + "useVpc": 'on', + "serviceProviderList": { + "Vpn": 'VpcVirtualRouter', + "Dhcp": 'VpcVirtualRouter', + "Dns": 'VpcVirtualRouter', + "SourceNat": 'VpcVirtualRouter', + "PortForwarding": 'VpcVirtualRouter', + "Lb": 'VpcVirtualRouter', + "UserData": 'VpcVirtualRouter', + "StaticNat": 'VpcVirtualRouter', + "NetworkACL": 'VpcVirtualRouter' + }, + "serviceCapabilityList": { + "SourceNat": {"SupportedSourceNatTypes": "peraccount"}, + "Lb": {"lbSchemes": "public", "SupportedLbIsolation": "dedicated"} + }, + }, + "network_off_netscaler": { + "name": 'Network offering-netscaler', + "displaytext": 'Network offering-netscaler', + "guestiptype": 'Isolated', + "supportedservices": 'Dhcp,Dns,SourceNat,PortForwarding,Vpn,Lb,UserData,StaticNat', + "traffictype": 'GUEST', + "availability": 'Optional', + "useVpc": 'on', + "serviceProviderList": { + "Dhcp": 'VpcVirtualRouter', + "Dns": 'VpcVirtualRouter', + "SourceNat": 'VpcVirtualRouter', + "PortForwarding": 'VpcVirtualRouter', + "Vpn": 'VpcVirtualRouter', + "Lb": 'Netscaler', + "UserData": 'VpcVirtualRouter', + "StaticNat": 'VpcVirtualRouter', + }, + "serviceCapabilityList": { + "SourceNat": {"SupportedSourceNatTypes": "peraccount"}, + "Lb": {"lbSchemes": "public", "SupportedLbIsolation": "dedicated"} + }, + }, + "network_off_shared": { + "name": 'Shared Network offering', + "displaytext": 'Shared Network offering', + "guestiptype": 'Shared', + "traffictype": 'GUEST', + "availability": 'Optional', + "useVpc": 'on', + "specifyIpRanges": True, + "specifyVlan": True + }, + "vpc_offering": { + "name": 'VPC off', + "displaytext": 'VPC off', + "supportedservices": 'Dhcp,Dns,SourceNat,PortForwarding,Vpn,Lb,UserData,StaticNat', + }, + "vpc": { + "name": "TestVPC", + "displaytext": "TestVPC", + "cidr": '10.0.0.1/24' + }, + "network": { + "name": "Test Network", + "displaytext": "Test Network", + "netmask": '255.255.255.0' + }, + "lbrule": { + "name": "SSH", + "alg": "leastconn", + # Algorithm used for load balancing + "privateport": 22, + "publicport": 2222, + "openfirewall": False, + "startport": 22, + "endport": 2222, + "protocol": "TCP", + "cidrlist": '0.0.0.0/0', + }, + "natrule": { + "privateport": 22, + "publicport": 22, + "startport": 22, + "endport": 22, + "protocol": "TCP", + "cidrlist": '0.0.0.0/0', + }, + "fw_rule": { + "startport": 1, + "endport": 6000, + "cidr": '0.0.0.0/0', + # Any network (For creating FW rule) + "protocol": "TCP" + }, + "http_rule": { + "startport": 80, + "endport": 80, + "cidrlist": '0.0.0.0/0', + "protocol": "ICMP" + }, + "virtual_machine": { + "displayname": "Test VM", + "username": "root", + "password": "password", + "ssh_port": 22, + "hypervisor": 'XenServer', + # Hypervisor type should be same as + # hypervisor type of cluster + "privateport": 22, + "publicport": 22, + "protocol": 'TCP', + }, + "ostype": 'CentOS 5.3 (64-bit)', + # Cent OS 5.3 (64 bit) + "sleep": 60, + "timeout": 10, + } class TestVPCNetwork(cloudstackTestCase): From 6607b1e5f1da3e738babcc557f40c30b932ed565 Mon Sep 17 00:00:00 2001 From: Prasanna Santhanam Date: Mon, 10 Jun 2013 14:52:47 +0530 Subject: [PATCH 22/30] Adding serviceCapability LB:Public for VPC Signed-off-by: Prasanna Santhanam --- .../component/test_vpc_vm_life_cycle.py | 1095 +++-------------- 1 file changed, 156 insertions(+), 939 deletions(-) diff --git a/test/integration/component/test_vpc_vm_life_cycle.py b/test/integration/component/test_vpc_vm_life_cycle.py index 44f72880f39..1fd2724a00b 100644 --- a/test/integration/component/test_vpc_vm_life_cycle.py +++ b/test/integration/component/test_vpc_vm_life_cycle.py @@ -26,8 +26,6 @@ from marvin.integration.lib.utils import * from marvin.integration.lib.base import * from marvin.integration.lib.common import * from marvin.remoteSSHClient import remoteSSHClient -import datetime - class Services: """Test VM life cycle in VPC network services @@ -35,155 +33,159 @@ class Services: def __init__(self): self.services = { - "account": { - "email": "test@test.com", - "firstname": "Test", - "lastname": "User", - "username": "test", - # Random characters are appended for unique - # username - "password": "password", - }, - "service_offering": { - "name": "Tiny Instance", - "displaytext": "Tiny Instance", - "cpunumber": 1, - "cpuspeed": 100, - "memory": 128, - }, - "service_offering_1": { - "name": "Tiny Instance- tagged host 1", - "displaytext": "Tiny off-tagged host2", - "cpunumber": 1, - "cpuspeed": 100, - "memory": 128, - "tags": "HOST_TAGS_HERE" - }, - "service_offering_2": { - "name": "Tiny Instance- tagged host 2", - "displaytext": "Tiny off-tagged host2", - "cpunumber": 1, - "cpuspeed": 100, - "memory": 128, - "tags": "HOST_TAGS_HERE" - }, - "network_offering": { - "name": 'VPC Network offering', - "displaytext": 'VPC Network off', - "guestiptype": 'Isolated', - "supportedservices": 'Dhcp,Dns,SourceNat,PortForwarding,Lb,UserData,StaticNat,NetworkACL', - "traffictype": 'GUEST', - "availability": 'Optional', - "useVpc": 'on', - "serviceProviderList": { - "Dhcp": 'VpcVirtualRouter', - "Dns": 'VpcVirtualRouter', - "SourceNat": 'VpcVirtualRouter', - "PortForwarding": 'VpcVirtualRouter', - "Lb": 'VpcVirtualRouter', - "UserData": 'VpcVirtualRouter', - "StaticNat": 'VpcVirtualRouter', - "NetworkACL": 'VpcVirtualRouter' - }, - }, - "network_offering_no_lb": { - "name": 'VPC Network offering', - "displaytext": 'VPC Network off', - "guestiptype": 'Isolated', - "supportedservices": 'Dhcp,Dns,SourceNat,PortForwarding,UserData,StaticNat,NetworkACL', - "traffictype": 'GUEST', - "availability": 'Optional', - "useVpc": 'on', - "serviceProviderList": { - "Dhcp": 'VpcVirtualRouter', - "Dns": 'VpcVirtualRouter', - "SourceNat": 'VpcVirtualRouter', - "PortForwarding": 'VpcVirtualRouter', - "UserData": 'VpcVirtualRouter', - "StaticNat": 'VpcVirtualRouter', - "NetworkACL": 'VpcVirtualRouter' - }, - }, - "network_off_shared": { - "name": 'Shared Network offering', - "displaytext": 'Shared Network offering', - "guestiptype": 'Shared', - "traffictype": 'GUEST', - "availability": 'Optional', - "useVpc": 'on', - "specifyIpRanges": True, - "specifyVlan": True - }, - "vpc_offering": { - "name": 'VPC off', - "displaytext": 'VPC off', - "supportedservices": 'Dhcp,Dns,SourceNat,PortForwarding,Lb,UserData,StaticNat', - }, - "vpc": { - "name": "TestVPC", - "displaytext": "TestVPC", - "cidr": '10.0.0.1/24' - }, - "network": { - "name": "Test Network", - "displaytext": "Test Network", - "netmask": '255.255.255.0', - "limit": 5, - # Max networks allowed as per hypervisor - # Xenserver -> 5, VMWare -> 9 - }, - "lbrule": { - "name": "SSH", - "alg": "leastconn", - # Algorithm used for load balancing - "privateport": 22, - "publicport": 2222, - "openfirewall": False, - "startport": 2222, - "endport": 2222, - "protocol": "TCP", - "cidrlist": '0.0.0.0/0', - }, - "natrule": { - "privateport": 22, - "publicport": 22, - "startport": 22, - "endport": 22, - "protocol": "TCP", - "cidrlist": '0.0.0.0/0', - }, - "fw_rule": { - "startport": 1, - "endport": 6000, - "cidr": '0.0.0.0/0', - # Any network (For creating FW rule) - "protocol": "TCP" - }, - "http_rule": { - "startport": 80, - "endport": 80, - "cidrlist": '0.0.0.0/0', - "protocol": "ICMP" - }, - "virtual_machine": { - "displayname": "Test VM", - "username": "root", - "password": "password", - "ssh_port": 22, - "hypervisor": 'XenServer', - # Hypervisor type should be same as - # hypervisor type of cluster - "privateport": 22, - "publicport": 22, - "protocol": 'TCP', - "userdata": 'This is sample data', - }, - "ostype": 'CentOS 5.3 (64-bit)', - # Cent OS 5.3 (64 bit) - "sleep": 60, - "timeout": 10, - "mode": 'advanced' - } + "account": { + "email": "test@test.com", + "firstname": "Test", + "lastname": "User", + "username": "test", + # Random characters are appended for unique + # username + "password": "password", + }, + "service_offering": { + "name": "Tiny Instance", + "displaytext": "Tiny Instance", + "cpunumber": 1, + "cpuspeed": 100, + "memory": 128, + }, + "service_offering_1": { + "name": "Tiny Instance- tagged host 1", + "displaytext": "Tiny off-tagged host2", + "cpunumber": 1, + "cpuspeed": 100, + "memory": 128, + "tags": "HOST_TAGS_HERE" + }, + "service_offering_2": { + "name": "Tiny Instance- tagged host 2", + "displaytext": "Tiny off-tagged host2", + "cpunumber": 1, + "cpuspeed": 100, + "memory": 128, + "tags": "HOST_TAGS_HERE" + }, + "network_offering": { + "name": 'VPC Network offering', + "displaytext": 'VPC Network off', + "guestiptype": 'Isolated', + "supportedservices": 'Dhcp,Dns,SourceNat,PortForwarding,Lb,UserData,StaticNat,NetworkACL', + "traffictype": 'GUEST', + "availability": 'Optional', + "useVpc": 'on', + "serviceProviderList": { + "Dhcp": 'VpcVirtualRouter', + "Dns": 'VpcVirtualRouter', + "SourceNat": 'VpcVirtualRouter', + "PortForwarding": 'VpcVirtualRouter', + "Lb": 'VpcVirtualRouter', + "UserData": 'VpcVirtualRouter', + "StaticNat": 'VpcVirtualRouter', + "NetworkACL": 'VpcVirtualRouter' + }, + "serviceCapabilityList": { + "SourceNat": {"SupportedSourceNatTypes": "peraccount"}, + "Lb": {"lbSchemes": "public", "SupportedLbIsolation": "dedicated"} + }, + }, + "network_offering_no_lb": { + "name": 'VPC Network offering no LB', + "displaytext": 'VPC Network off no LB', + "guestiptype": 'Isolated', + "supportedservices": 'Dhcp,Dns,SourceNat,PortForwarding,UserData,StaticNat,NetworkACL', + "traffictype": 'GUEST', + "availability": 'Optional', + "useVpc": 'on', + "serviceProviderList": { + "Dhcp": 'VpcVirtualRouter', + "Dns": 'VpcVirtualRouter', + "SourceNat": 'VpcVirtualRouter', + "PortForwarding": 'VpcVirtualRouter', + "UserData": 'VpcVirtualRouter', + "StaticNat": 'VpcVirtualRouter', + "NetworkACL": 'VpcVirtualRouter' + }, + }, + "network_off_shared": { + "name": 'Shared Network offering', + "displaytext": 'Shared Network offering', + "guestiptype": 'Shared', + "traffictype": 'GUEST', + "availability": 'Optional', + "useVpc": 'on', + "specifyIpRanges": True, + "specifyVlan": True + }, + "vpc_offering": { + "name": 'VPC off', + "displaytext": 'VPC off', + "supportedservices": 'Dhcp,Dns,SourceNat,PortForwarding,Lb,UserData,StaticNat', + }, + "vpc": { + "name": "TestVPC", + "displaytext": "TestVPC", + "cidr": '10.0.0.1/24' + }, + "network": { + "name": "Test Network", + "displaytext": "Test Network", + "netmask": '255.255.255.0', + "limit": 5, + # Max networks allowed as per hypervisor + # Xenserver -> 5, VMWare -> 9 + }, + "lbrule": { + "name": "SSH", + "alg": "leastconn", + # Algorithm used for load balancing + "privateport": 22, + "publicport": 2222, + "openfirewall": False, + "startport": 2222, + "endport": 2222, + "protocol": "TCP", + "cidrlist": '0.0.0.0/0', + }, + "natrule": { + "privateport": 22, + "publicport": 22, + "startport": 22, + "endport": 22, + "protocol": "TCP", + "cidrlist": '0.0.0.0/0', + }, + "fw_rule": { + "startport": 1, + "endport": 6000, + "cidr": '0.0.0.0/0', + # Any network (For creating FW rule) + "protocol": "TCP" + }, + "http_rule": { + "startport": 80, + "endport": 80, + "cidrlist": '0.0.0.0/0', + "protocol": "ICMP" + }, + "virtual_machine": { + "displayname": "Test VM", + "username": "root", + "password": "password", + "ssh_port": 22, + "hypervisor": 'XenServer', + # Hypervisor type should be same as + # hypervisor type of cluster + "privateport": 22, + "publicport": 22, + "protocol": 'TCP', + "userdata": 'This is sample data', + }, + "ostype": 'CentOS 5.3 (64-bit)', + # Cent OS 5.3 (64 bit) + "sleep": 60, + "timeout": 10, + "mode": 'advanced' + } class TestVMLifeCycleVPC(cloudstackTestCase): @@ -914,7 +916,7 @@ class TestVMLifeCycleVPC(cloudstackTestCase): ) return - +@unittest.skip("debugging") class TestVMLifeCycleSharedNwVPC(cloudstackTestCase): @classmethod @@ -1722,6 +1724,7 @@ class TestVMLifeCycleSharedNwVPC(cloudstackTestCase): return +@unittest.skip("debugging") class TestVMLifeCycleBothIsolated(cloudstackTestCase): @classmethod @@ -2060,6 +2063,7 @@ class TestVMLifeCycleBothIsolated(cloudstackTestCase): return +@unittest.skip("debugging") class TestVMLifeCycleStoppedVPCVR(cloudstackTestCase): @classmethod @@ -2804,791 +2808,4 @@ class TestVMLifeCycleStoppedVPCVR(cloudstackTestCase): return -class TestVMLifeCycleDiffHosts(cloudstackTestCase): - @classmethod - def setUpClass(cls): - cls.api_client = super( - TestVMLifeCycleDiffHosts, - cls - ).getClsTestClient().getApiClient() - cls.services = Services().services - # Get Zone, Domain and templates - cls.domain = get_domain(cls.api_client, cls.services) - cls.zone = get_zone(cls.api_client, cls.services) - cls.template = get_template( - cls.api_client, - cls.zone.id, - cls.services["ostype"] - ) - cls.services["virtual_machine"]["zoneid"] = cls.zone.id - cls.services["virtual_machine"]["template"] = cls.template.id - - cls.service_offering_1 = ServiceOffering.create( - cls.api_client, - cls.services["service_offering_1"] - ) - cls.service_offering_2 = ServiceOffering.create( - cls.api_client, - cls.services["service_offering_2"] - ) - cls.vpc_off = VpcOffering.create( - cls.api_client, - cls.services["vpc_offering"] - ) - cls.vpc_off.update(cls.api_client, state='Enabled') - - cls.account = Account.create( - cls.api_client, - cls.services["account"], - admin=True, - domainid=cls.domain.id - ) - - cls.vpc_off = VpcOffering.create( - cls.api_client, - cls.services["vpc_offering"] - ) - - cls.vpc_off.update(cls.api_client, state='Enabled') - - cls.services["vpc"]["cidr"] = '10.1.1.1/16' - cls.vpc = VPC.create( - cls.api_client, - cls.services["vpc"], - vpcofferingid=cls.vpc_off.id, - zoneid=cls.zone.id, - account=cls.account.name, - domainid=cls.account.domainid - ) - - cls.nw_off = NetworkOffering.create( - cls.api_client, - cls.services["network_offering"], - conservemode=False - ) - # Enable Network offering - cls.nw_off.update(cls.api_client, state='Enabled') - - # Creating network using the network offering created - cls.network_1 = Network.create( - cls.api_client, - cls.services["network"], - accountid=cls.account.name, - domainid=cls.account.domainid, - networkofferingid=cls.nw_off.id, - zoneid=cls.zone.id, - gateway='10.1.1.1', - vpcid=cls.vpc.id - ) - cls.nw_off_no_lb = NetworkOffering.create( - cls.api_client, - cls.services["network_offering_no_lb"], - conservemode=False - ) - # Enable Network offering - cls.nw_off_no_lb.update(cls.api_client, state='Enabled') - - # Creating network using the network offering created - cls.network_2 = Network.create( - cls.api_client, - cls.services["network"], - accountid=cls.account.name, - domainid=cls.account.domainid, - networkofferingid=cls.nw_off_no_lb.id, - zoneid=cls.zone.id, - gateway='10.1.2.1', - vpcid=cls.vpc.id - ) - # Spawn an instance in that network - cls.vm_1 = VirtualMachine.create( - cls.api_client, - cls.services["virtual_machine"], - accountid=cls.account.name, - domainid=cls.account.domainid, - serviceofferingid=cls.service_offering_1.id, - networkids=[str(cls.network_1.id)] - ) - # Spawn an instance in that network - cls.vm_2 = VirtualMachine.create( - cls.api_client, - cls.services["virtual_machine"], - accountid=cls.account.name, - domainid=cls.account.domainid, - serviceofferingid=cls.service_offering_1.id, - networkids=[str(cls.network_1.id)] - ) - cls.vm_3 = VirtualMachine.create( - cls.api_client, - cls.services["virtual_machine"], - accountid=cls.account.name, - domainid=cls.account.domainid, - serviceofferingid=cls.service_offering_2.id, - networkids=[str(cls.network_2.id)] - ) - - cls.public_ip_1 = PublicIPAddress.create( - cls.api_client, - accountid=cls.account.name, - zoneid=cls.zone.id, - domainid=cls.account.domainid, - networkid=cls.network_1.id, - vpcid=cls.vpc.id - ) - cls.lb_rule = LoadBalancerRule.create( - cls.api_client, - cls.services["lbrule"], - ipaddressid=cls.public_ip_1.ipaddress.id, - accountid=cls.account.name, - networkid=cls.network_1.id, - vpcid=cls.vpc.id, - domainid=cls.account.domainid - ) - cls.lb_rule.assign(cls.api_client, [cls.vm_1, cls.vm_2]) - - cls.public_ip_2 = PublicIPAddress.create( - cls.api_client, - accountid=cls.account.name, - zoneid=cls.zone.id, - domainid=cls.account.domainid, - networkid=cls.network_1.id, - vpcid=cls.vpc.id - ) - - cls.nat_rule = NATRule.create( - cls.api_client, - cls.vm_1, - cls.services["natrule"], - ipaddressid=cls.public_ip_2.ipaddress.id, - openfirewall=False, - networkid=cls.network_1.id, - vpcid=cls.vpc.id - ) - - # Opening up the ports in VPC - cls.nwacl_nat = NetworkACL.create( - cls.api_client, - networkid=cls.network_1.id, - services=cls.services["natrule"], - traffictype='Ingress' - ) - - cls.nwacl_lb = NetworkACL.create( - cls.api_client, - networkid=cls.network_1.id, - services=cls.services["lbrule"], - traffictype='Ingress' - ) - - cls.nwacl_internet = NetworkACL.create( - cls.api_client, - networkid=cls.network_1.id, - services=cls.services["http_rule"], - traffictype='Egress' - ) - cls._cleanup = [ - cls.service_offering_1, - cls.service_offering_2, - cls.nw_off, - cls.nw_off_no_lb, - ] - return - - @classmethod - def tearDownClass(cls): - try: - cls.account.delete(cls.api_client) - wait_for_cleanup(cls.api_client, ["account.cleanup.interval"]) - #Cleanup resources used - cleanup_resources(cls.api_client, cls._cleanup) - - # Waiting for network cleanup to delete vpc offering - wait_for_cleanup(cls.api_client, ["network.gc.wait", - "network.gc.interval"]) - cls.vpc_off.delete(cls.api_client) - except Exception as e: - raise Exception("Warning: Exception during cleanup : %s" % e) - return - - def setUp(self): - - self.apiclient = self.testClient.getApiClient() - self.dbclient = self.testClient.getDbConnection() - self.debug("Check the status of VPC virtual router") - routers = Router.list( - self.apiclient, - networkid=self.network_1.id, - listall=True - ) - if not isinstance(routers, list): - raise Exception("No response from list routers API") - - self.router = routers[0] - if self.router.state == "Running": - Router.stop(self.apiclient, id=self.router.id) - - self.cleanup = [] - return - - def tearDown(self): - try: - #Clean up, terminate the created network offerings - cleanup_resources(self.apiclient, self.cleanup) - wait_for_cleanup(self.apiclient, [ - "network.gc.interval", - "network.gc.wait"]) - - except Exception as e: - raise Exception("Warning: Exception during cleanup : %s" % e) - return - - def validate_vm_deployment(self): - """Validates VM deployment on different hosts""" - - vms = VirtualMachine.list( - self.apiclient, - account=self.account.name, - domainid=self.account.domainid, - networkid=self.network_1.id, - listall=True - ) - self.assertEqual( - isinstance(vms, list), - True, - "List VMs shall return a valid response" - ) - host_1 = vms[0].hostid - self.debug("Host for network 1: %s" % vms[0].hostid) - - vms = VirtualMachine.list( - self.apiclient, - account=self.account.name, - domainid=self.account.domainid, - networkid=self.network_2.id, - listall=True - ) - self.assertEqual( - isinstance(vms, list), - True, - "List VMs shall return a valid response" - ) - host_2 = vms[0].hostid - self.debug("Host for network 2: %s" % vms[0].hostid) - - self.assertNotEqual( - host_1, - host_2, - "Both the virtual machines should be deployed on diff hosts " - ) - return - - def validate_vpc_offering(self, vpc_offering): - """Validates the VPC offering""" - - self.debug("Check if the VPC offering is created successfully?") - vpc_offs = VpcOffering.list( - self.apiclient, - id=vpc_offering.id - ) - self.assertEqual( - isinstance(vpc_offs, list), - True, - "List VPC offerings should return a valid list" - ) - self.assertEqual( - vpc_offering.name, - vpc_offs[0].name, - "Name of the VPC offering should match with listVPCOff data" - ) - self.debug( - "VPC offering is created successfully - %s" % - vpc_offering.name) - return - - def validate_vpc_network(self, network, state=None): - """Validates the VPC network""" - - self.debug("Check if the VPC network is created successfully?") - vpc_networks = VPC.list( - self.apiclient, - id=network.id - ) - self.assertEqual( - isinstance(vpc_networks, list), - True, - "List VPC network should return a valid list" - ) - self.assertEqual( - network.name, - vpc_networks[0].name, - "Name of the VPC network should match with listVPC data" - ) - if state: - self.assertEqual( - vpc_networks[0].state, - state, - "VPC state should be '%s'" % state - ) - self.debug("VPC network validated - %s" % network.name) - return - - def validate_network_rules(self): - """Validates if the network rules work properly or not?""" - try: - ssh_1 = self.vm_1.get_ssh_client( - ipaddress=self.public_ip_1.ipaddress.ipaddress) - self.debug("SSH into VM is successfully") - - self.debug("Verifying if we can ping to outside world from VM?") - # Ping to outsite world - res = ssh_1.execute("ping -c 1 www.google.com") - # res = 64 bytes from maa03s17-in-f20.1e100.net (74.125.236.212): - # icmp_req=1 ttl=57 time=25.9 ms - # --- www.l.google.com ping statistics --- - # 1 packets transmitted, 1 received, 0% packet loss, time 0ms - # rtt min/avg/max/mdev = 25.970/25.970/25.970/0.000 ms - except Exception as e: - self.fail("Failed to SSH into VM - %s, %s" % - (self.public_ip_1.ipaddress.ipaddress, e)) - - result = str(res) - self.assertEqual( - result.count("1 received"), - 1, - "Ping to outside world from VM should be successful" - ) - - self.debug("Checking if we can SSH into VM_1?") - try: - ssh_2 = self.vm_1.get_ssh_client( - ipaddress=self.public_ip_2.ipaddress.ipaddress) - self.debug("SSH into VM is successfully") - - self.debug("Verifying if we can ping to outside world from VM?") - res = ssh_2.execute("ping -c 1 www.google.com") - except Exception as e: - self.fail("Failed to SSH into VM - %s, %s" % - (self.public_ip_2.ipaddress.ipaddress, e)) - - result = str(res) - self.assertEqual( - result.count("1 received"), - 1, - "Ping to outside world from VM should be successful" - ) - return - - @attr(tags=["advanced", "intervlan"]) - def test_01_deploy_instance_in_network(self): - """ Test deploy an instance in VPC networks - """ - - # Validate the following - # 1. Create a VPC with cidr - 10.1.1.1/16 - # 2. Add network1(10.1.1.1/24) and network2(10.1.2.1/24) to this VPC. - # Steps: - # 1. Deploy vm1 and vm2 in network1 and vm3 and vm4 in network2 using - # the default CentOS 6.2 Template - - self.validate_vm_deployment() - self.debug("Check if deployed VMs are in running state?") - vms = VirtualMachine.list( - self.apiclient, - account=self.account.name, - domainid=self.account.domainid, - listall=True - ) - self.assertEqual( - isinstance(vms, list), - True, - "List VMs should return a valid response" - ) - for vm in vms: - self.debug("VM name: %s, VM state: %s" % (vm.name, vm.state)) - self.assertEqual( - vm.state, - "Running", - "Vm state should be running for each VM deployed" - ) - return - - @attr(tags=["advanced", "intervlan"]) - def test_02_stop_instance_in_network(self): - """ Test stop an instance in VPC networks - """ - - # Validate the following - # 1. Stop the virtual machines. - # 2. Rules should be still configured on virtual router. - - self.debug("Stopping the virtual machines in account: %s" % - self.account.name) - try: - self.vm_1.stop(self.apiclient) - self.vm_2.stop(self.apiclient) - except Exception as e: - self.fail("Failed to stop the virtual instances, %s" % e) - - # Check if the network rules still exists after Vm stop - self.debug("Checking if NAT rules ") - nat_rules = NATRule.list( - self.apiclient, - id=self.nat_rule.id, - listall=True - ) - self.assertEqual( - isinstance(nat_rules, list), - True, - "List NAT rules shall return a valid list" - ) - - lb_rules = LoadBalancerRule.list( - self.apiclient, - id=self.lb_rule.id, - listall=True - ) - self.assertEqual( - isinstance(lb_rules, list), - True, - "List LB rules shall return a valid list" - ) - return - - @attr(tags=["advanced", "intervlan"]) - def test_03_start_instance_in_network(self): - """ Test start an instance in VPC networks - """ - - # Validate the following - # 1. Start the virtual machines. - # 2. Vm should be started successfully. - # 3. Make sure that all the PF,LB and Static NAT rules on this VM - # works as expected. - # 3. Make sure that we are able to access google.com from this user Vm - - self.debug("Starting the virtual machines in account: %s" % - self.account.name) - try: - self.vm_1.start(self.apiclient) - self.vm_2.start(self.apiclient) - except Exception as e: - self.fail("Failed to start the virtual instances, %s" % e) - self.debug("Validating if the network rules work properly or not?") - self.validate_network_rules() - return - - @attr(tags=["advanced", "intervlan"]) - def test_04_reboot_instance_in_network(self): - """ Test reboot an instance in VPC networks - """ - - # Validate the following - # 1. Reboot the virtual machines. - # 2. Vm should be started successfully. - # 3. Make sure that all the PF,LB and Static NAT rules on this VM - # works as expected. - # 3. Make sure that we are able to access google.com from this user Vm - - self.debug("Validating if the network rules work properly or not?") - self.validate_network_rules() - - self.debug("Starting the virtual machines in account: %s" % - self.account.name) - try: - self.vm_1.reboot(self.apiclient) - self.vm_2.reboot(self.apiclient) - except Exception as e: - self.fail("Failed to reboot the virtual instances, %s" % e) - - self.debug("Validating if the network rules work properly or not?") - self.validate_network_rules() - return - - @attr(tags=["advanced", "intervlan"]) - def test_05_destroy_instance_in_network(self): - """ Test destroy an instance in VPC networks - """ - - # Validate the following - # 1. Destory the virtual machines. - # 2. Rules should be still configured on virtual router. - - self.debug("Validating if the network rules work properly or not?") - self.validate_network_rules() - - self.debug("Destroying the virtual machines in account: %s" % - self.account.name) - try: - self.vm_1.delete(self.apiclient) - self.vm_2.delete(self.apiclient) - except Exception as e: - self.fail("Failed to stop the virtual instances, %s" % e) - - # Check if the network rules still exists after Vm stop - self.debug("Checking if NAT rules ") - nat_rules = NATRule.list( - self.apiclient, - id=self.nat_rule.id, - listall=True - ) - self.assertEqual( - isinstance(nat_rules, list), - True, - "List NAT rules shall return a valid list" - ) - - lb_rules = LoadBalancerRule.list( - self.apiclient, - id=self.lb_rule.id, - listall=True - ) - self.assertEqual( - isinstance(lb_rules, list), - True, - "List LB rules shall return a valid list" - ) - return - - @attr(tags=["advanced", "intervlan"]) - def test_06_recover_instance_in_network(self): - """ Test recover an instance in VPC networks - """ - - # Validate the following - # 1. Recover the virtual machines. - # 2. Vm should be in stopped state. State both the instances - # 3. Make sure that all the PF,LB and Static NAT rules on this VM - # works as expected. - # 3. Make sure that we are able to access google.com from this user Vm - - self.debug("Recovering the expunged virtual machines in account: %s" % - self.account.name) - try: - self.vm_1.recover(self.apiclient) - self.vm_2.recover(self.apiclient) - except Exception as e: - self.fail("Failed to recover the virtual instances, %s" % e) - - self.debug("Starting the two instances..") - try: - self.vm_1.start(self.apiclient) - self.vm_2.start(self.apiclient) - except Exception as e: - self.fail("Failed to start the instances, %s" % e) - - self.debug("Validating if the network rules work properly or not?") - self.validate_network_rules() - return - - @attr(tags=["advanced", "intervlan"]) - def test_07_migrate_instance_in_network(self): - """ Test migrate an instance in VPC networks - """ - - # Validate the following - # 1. Migrate the virtual machines to other hosts - # 2. Vm should be in stopped state. State both the instances - # 3. Make sure that all the PF,LB and Static NAT rules on this VM - # works as expected. - # 3. Make sure that we are able to access google.com from this user Vm - - self.debug("Checking if the host is available for migration?") - hosts = Host.list( - self.apiclient, - zoneid=self.zone.id, - type='Routing' - ) - - self.assertEqual( - isinstance(hosts, list), - True, - "List hosts should return a valid list" - ) - if len(hosts) < 2: - raise unittest.SkipTest( - "No host available for migration. Test requires atleast 2 hosts") - - # Remove the host of current VM from the hosts list - hosts[:] = [host for host in hosts if host.id != self.vm_1.hostid] - - host = hosts[0] - - self.debug("Validating if the network rules work properly or not?") - self.validate_network_rules() - - self.debug("Migrating VM-ID: %s to Host: %s" % ( - self.vm_1.id, - host.id - )) - - try: - self.vm_1.migrate(self.apiclient, hostid=host.id) - except Exception as e: - self.fail("Failed to migrate instance, %s" % e) - - self.debug("Validating if the network rules work properly or not?") - self.validate_network_rules() - return - - @attr(tags=["advanced", "intervlan"]) - def test_08_user_data(self): - """ Test user data in virtual machines - """ - - # Validate the following - # 1. Create a VPC with cidr - 10.1.1.1/16 - # 2. Add network1(10.1.1.1/24) and network2(10.1.2.1/24) to this VPC. - # 3. Deploy a vm in network1 and a vm in network2 using userdata - # Steps - # 1.Query for the user data for both the user vms from both networks - # User should be able to query the user data for the vms belonging to - # both the networks from the VR - - try: - ssh = self.vm_1.get_ssh_client( - ipaddress=self.public_ip_1.ipaddress.ipaddress) - self.debug("SSH into VM is successfully") - except Exception as e: - self.fail("Failed to SSH into instance") - - # Find router associated with user account - routers = Router.list( - self.apiclient, - zoneid=self.zone.id, - listall=True - ) - self.assertEqual( - isinstance(routers, list), - True, - "Check list response returns a valid list" - ) - router = routers[0] - self.debug("check the userdata with that of present in router") - try: - cmds = [ - "wget http://%s/latest/user-data" % router.guestipaddress, - "cat user-data", - ] - for c in cmds: - result = ssh.execute(c) - self.debug("%s: %s" % (c, result)) - except Exception as e: - self.fail("Failed to SSH in Virtual machine: %s" % e) - - res = str(result) - self.assertEqual( - res.count( - self.services["virtual_machine"]["userdata"]), - 1, - "Verify user data from router" - ) - return - - @attr(tags=["advanced", "intervlan"]) - def test_09_meta_data(self): - """ Test meta data in virtual machines - """ - - # Validate the following - # 1. Create a VPC with cidr - 10.1.1.1/16 - # 2. Add network1(10.1.1.1/24) and network2(10.1.2.1/24) to this VPC. - # 3. Deploy a vm in network1 and a vm in network2 using userdata - # Steps - # 1.Query for the meta data for both the user vms from both networks - # User should be able to query the user data for the vms belonging to - # both the networks from the VR - - try: - ssh = self.vm_1.get_ssh_client( - ipaddress=self.public_ip_1.ipaddress.ipaddress) - self.debug("SSH into VM is successfully") - except Exception as e: - self.fail("Failed to SSH into instance") - - # Find router associated with user account - routers = Router.list( - self.apiclient, - zoneid=self.zone.id, - listall=True - ) - self.assertEqual( - isinstance(routers, list), - True, - "Check list response returns a valid list" - ) - router = routers[0] - self.debug("check the metadata with that of present in router") - try: - cmds = [ - "wget http://%s/latest/meta-data" % router.guestipaddress, - "cat user-data", - ] - for c in cmds: - result = ssh.execute(c) - self.debug("%s: %s" % (c, result)) - except Exception as e: - self.fail("Failed to SSH in Virtual machine: %s" % e) - - res = str(result) - self.assertNotEqual( - res, - None, - "Meta data should be returned from router" - ) - return - - @attr(tags=["advanced", "intervlan"]) - def test_10_expunge_instance_in_network(self): - """ Test expunge an instance in VPC networks - """ - - # Validate the following - # 1. Recover the virtual machines. - # 2. Vm should be in stopped state. State both the instances - # 3. Make sure that all the PF,LB and Static NAT rules on this VM - # works as expected. - # 3. Make sure that we are able to access google.com from this user Vm - - self.debug("Validating if the network rules work properly or not?") - self.validate_network_rules() - - self.debug("Delete virtual machines in account: %s" % - self.account.name) - try: - self.vm_1.delete(self.apiclient) - self.vm_2.delete(self.apiclient) - except Exception as e: - self.fail("Failed to destroy the virtual instances, %s" % e) - - self.debug( - "Waiting for expunge interval to cleanup the network and VMs") - - wait_for_cleanup( - self.apiclient, - ["expunge.interval", "expunge.delay"] - ) - - # Check if the network rules still exists after Vm stop - self.debug("Checking if NAT rules ") - nat_rules = NATRule.list( - self.apiclient, - id=self.nat_rule.id, - listall=True - ) - self.assertEqual( - nat_rules, - None, - "List NAT rules should not return anything" - ) - - lb_rules = LoadBalancerRule.list( - self.apiclient, - id=self.lb_rule.id, - listall=True - ) - self.assertEqual( - lb_rules, - None, - "List LB rules should not return anything" - ) - return From 4a99f4758136b2e824cc5e6be4aa0cd74c826d6c Mon Sep 17 00:00:00 2001 From: Prasanna Santhanam Date: Mon, 10 Jun 2013 15:40:52 +0530 Subject: [PATCH 23/30] Fixes to test_vpc_lifecycle - Removing redundant creation of VPC offerings - Removing cleanup based on configurations.GC should happen by default - Speed up the run by not waiting for complete gc. Signed-off-by: Prasanna Santhanam --- .../component/test_vpc_vm_life_cycle.py | 80 ++----------------- 1 file changed, 6 insertions(+), 74 deletions(-) diff --git a/test/integration/component/test_vpc_vm_life_cycle.py b/test/integration/component/test_vpc_vm_life_cycle.py index 1fd2724a00b..e198c79b88d 100644 --- a/test/integration/component/test_vpc_vm_life_cycle.py +++ b/test/integration/component/test_vpc_vm_life_cycle.py @@ -212,11 +212,6 @@ class TestVMLifeCycleVPC(cloudstackTestCase): cls.api_client, cls.services["service_offering"] ) - cls.vpc_off = VpcOffering.create( - cls.api_client, - cls.services["vpc_offering"] - ) - cls.vpc_off.update(cls.api_client, state='Enabled') cls.account = Account.create( cls.api_client, @@ -268,18 +263,6 @@ class TestVMLifeCycleVPC(cloudstackTestCase): ) # Enable Network offering cls.nw_off_no_lb.update(cls.api_client, state='Enabled') - - # Creating network using the network offering created - cls.network_2 = Network.create( - cls.api_client, - cls.services["network"], - accountid=cls.account.name, - domainid=cls.account.domainid, - networkofferingid=cls.nw_off_no_lb.id, - zoneid=cls.zone.id, - gateway='10.1.2.1', - vpcid=cls.vpc.id - ) # Spawn an instance in that network cls.vm_1 = VirtualMachine.create( cls.api_client, @@ -289,7 +272,6 @@ class TestVMLifeCycleVPC(cloudstackTestCase): serviceofferingid=cls.service_offering.id, networkids=[str(cls.network_1.id)] ) - # Spawn an instance in that network cls.vm_2 = VirtualMachine.create( cls.api_client, cls.services["virtual_machine"], @@ -298,15 +280,6 @@ class TestVMLifeCycleVPC(cloudstackTestCase): serviceofferingid=cls.service_offering.id, networkids=[str(cls.network_1.id)] ) - cls.vm_3 = VirtualMachine.create( - cls.api_client, - cls.services["virtual_machine"], - accountid=cls.account.name, - domainid=cls.account.domainid, - serviceofferingid=cls.service_offering.id, - networkids=[str(cls.network_2.id)] - ) - cls.public_ip_1 = PublicIPAddress.create( cls.api_client, accountid=cls.account.name, @@ -370,20 +343,15 @@ class TestVMLifeCycleVPC(cloudstackTestCase): cls.service_offering, cls.nw_off, cls.nw_off_no_lb, + cls.account ] return @classmethod def tearDownClass(cls): try: - cls.account.delete(cls.api_client) - wait_for_cleanup(cls.api_client, ["account.cleanup.interval"]) #Cleanup resources used cleanup_resources(cls.api_client, cls._cleanup) - - # Waiting for network cleanup to delete vpc offering - wait_for_cleanup(cls.api_client, ["network.gc.wait", "network.gc.interval"]) - cls.vpc_off.delete(cls.api_client) except Exception as e: raise Exception("Warning: Exception during cleanup : %s" % e) return @@ -398,10 +366,6 @@ class TestVMLifeCycleVPC(cloudstackTestCase): try: #Clean up, terminate the created network offerings cleanup_resources(self.apiclient, self.cleanup) - wait_for_cleanup(self.apiclient, [ - "network.gc.interval", - "network.gc.wait"]) - except Exception as e: raise Exception("Warning: Exception during cleanup : %s" % e) return @@ -1106,16 +1070,14 @@ class TestVMLifeCycleSharedNwVPC(cloudstackTestCase): cls.service_offering, cls.nw_off, cls.shared_nw_off, - cls.vpc_off + cls.vpc_off, + cls.account ] return @classmethod def tearDownClass(cls): try: - cls.account.delete(cls.api_client) - wait_for_cleanup(cls.api_client, ["account.cleanup.interval"]) - #Cleanup resources used cleanup_resources(cls.api_client, cls._cleanup) except Exception as e: raise Exception("Warning: Exception during cleanup : %s" % e) @@ -1129,12 +1091,7 @@ class TestVMLifeCycleSharedNwVPC(cloudstackTestCase): def tearDown(self): try: - #Clean up, terminate the created network offerings cleanup_resources(self.apiclient, self.cleanup) - wait_for_cleanup(self.apiclient, [ - "network.gc.interval", - "network.gc.wait"]) - except Exception as e: raise Exception("Warning: Exception during cleanup : %s" % e) return @@ -1822,16 +1779,14 @@ class TestVMLifeCycleBothIsolated(cloudstackTestCase): cls.service_offering, cls.nw_off, cls.nw_off_no_lb, - cls.vpc_off + cls.vpc_off, + cls.account ] return @classmethod def tearDownClass(cls): try: - cls.account.delete(cls.api_client) - wait_for_cleanup(cls.api_client, ["account.cleanup.interval"]) - #Cleanup resources used cleanup_resources(cls.api_client, cls._cleanup) except Exception as e: raise Exception("Warning: Exception during cleanup : %s" % e) @@ -1847,10 +1802,6 @@ class TestVMLifeCycleBothIsolated(cloudstackTestCase): try: #Clean up, terminate the created network offerings cleanup_resources(self.apiclient, self.cleanup) - wait_for_cleanup(self.apiclient, [ - "network.gc.interval", - "network.gc.wait"]) - except Exception as e: raise Exception("Warning: Exception during cleanup : %s" % e) return @@ -2088,11 +2039,6 @@ class TestVMLifeCycleStoppedVPCVR(cloudstackTestCase): cls.api_client, cls.services["service_offering"] ) - cls.vpc_off = VpcOffering.create( - cls.api_client, - cls.services["vpc_offering"] - ) - cls.vpc_off.update(cls.api_client, state='Enabled') cls.account = Account.create( cls.api_client, @@ -2246,21 +2192,14 @@ class TestVMLifeCycleStoppedVPCVR(cloudstackTestCase): cls.service_offering, cls.nw_off, cls.nw_off_no_lb, + cls.account ] return @classmethod def tearDownClass(cls): try: - cls.account.delete(cls.api_client) - wait_for_cleanup(cls.api_client, ["account.cleanup.interval"]) - #Cleanup resources used cleanup_resources(cls.api_client, cls._cleanup) - - # Waiting for network cleanup to delete vpc offering - wait_for_cleanup(cls.api_client, ["network.gc.wait", - "network.gc.interval"]) - cls.vpc_off.delete(cls.api_client) except Exception as e: raise Exception("Warning: Exception during cleanup : %s" % e) return @@ -2289,10 +2228,6 @@ class TestVMLifeCycleStoppedVPCVR(cloudstackTestCase): try: #Clean up, terminate the created network offerings cleanup_resources(self.apiclient, self.cleanup) - wait_for_cleanup(self.apiclient, [ - "network.gc.interval", - "network.gc.wait"]) - except Exception as e: raise Exception("Warning: Exception during cleanup : %s" % e) return @@ -2806,6 +2741,3 @@ class TestVMLifeCycleStoppedVPCVR(cloudstackTestCase): "List LB rules should not return anything" ) return - - - From 03e283c4b3ed0046a47777b9dc1ad4ab948254af Mon Sep 17 00:00:00 2001 From: Prasanna Santhanam Date: Mon, 10 Jun 2013 15:50:50 +0530 Subject: [PATCH 24/30] Unskip skipped tests Signed-off-by: Prasanna Santhanam --- test/integration/component/test_vpc_vm_life_cycle.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/test/integration/component/test_vpc_vm_life_cycle.py b/test/integration/component/test_vpc_vm_life_cycle.py index e198c79b88d..39fb533bfee 100644 --- a/test/integration/component/test_vpc_vm_life_cycle.py +++ b/test/integration/component/test_vpc_vm_life_cycle.py @@ -880,7 +880,6 @@ class TestVMLifeCycleVPC(cloudstackTestCase): ) return -@unittest.skip("debugging") class TestVMLifeCycleSharedNwVPC(cloudstackTestCase): @classmethod @@ -1681,7 +1680,6 @@ class TestVMLifeCycleSharedNwVPC(cloudstackTestCase): return -@unittest.skip("debugging") class TestVMLifeCycleBothIsolated(cloudstackTestCase): @classmethod @@ -2014,7 +2012,6 @@ class TestVMLifeCycleBothIsolated(cloudstackTestCase): return -@unittest.skip("debugging") class TestVMLifeCycleStoppedVPCVR(cloudstackTestCase): @classmethod From 63494f8dfd1f1633170262b1c8e14efce5bc3bb2 Mon Sep 17 00:00:00 2001 From: Prasanna Santhanam Date: Mon, 10 Jun 2013 16:37:41 +0530 Subject: [PATCH 25/30] rename the test methods to be more pythonic :) Signed-off-by: Prasanna Santhanam --- .../component/test_vpc_network_pfrules.py | 487 +++++++++--------- 1 file changed, 237 insertions(+), 250 deletions(-) diff --git a/test/integration/component/test_vpc_network_pfrules.py b/test/integration/component/test_vpc_network_pfrules.py index c0c2b86426c..92b04ad3f21 100644 --- a/test/integration/component/test_vpc_network_pfrules.py +++ b/test/integration/component/test_vpc_network_pfrules.py @@ -44,138 +44,139 @@ from marvin.integration.lib.common import (get_domain, class Services: """Test VPC network services - Port Forwarding Rules Test Data Class. """ + def __init__(self): self.services = { - "account": { - "email": "test@test.com", - "firstname": "Test", - "lastname": "User", - "username": "test", - # Random characters are appended for unique - # username - "password": "password", - }, - "host1":None, - "host2":None, - "service_offering": { - "name": "Tiny Instance", - "displaytext": "Tiny Instance", - "cpunumber": 1, - "cpuspeed": 1000, - "memory": 512, - }, - "network_offering": { - "name": 'VPC Network offering', - "displaytext": 'VPC Network off', - "guestiptype": 'Isolated', - "supportedservices": 'Vpn,Dhcp,Dns,SourceNat,PortForwarding,Lb,UserData,StaticNat,NetworkACL', - "traffictype": 'GUEST', - "availability": 'Optional', - "useVpc": 'on', - "serviceProviderList": { - "Vpn": 'VpcVirtualRouter', - "Dhcp": 'VpcVirtualRouter', - "Dns": 'VpcVirtualRouter', - "SourceNat": 'VpcVirtualRouter', - "PortForwarding": 'VpcVirtualRouter', - "Lb": 'VpcVirtualRouter', - "UserData": 'VpcVirtualRouter', - "StaticNat": 'VpcVirtualRouter', - "NetworkACL": 'VpcVirtualRouter' - }, - "servicecapabilitylist": { - }, - }, - "network_offering_no_lb": { - "name": 'VPC Network offering', - "displaytext": 'VPC Network off', - "guestiptype": 'Isolated', - "supportedservices": 'Dhcp,Dns,SourceNat,PortForwarding,UserData,StaticNat,NetworkACL', - "traffictype": 'GUEST', - "availability": 'Optional', - "useVpc": 'on', - "serviceProviderList": { - "Dhcp": 'VpcVirtualRouter', - "Dns": 'VpcVirtualRouter', - "SourceNat": 'VpcVirtualRouter', - "PortForwarding": 'VpcVirtualRouter', - "UserData": 'VpcVirtualRouter', - "StaticNat": 'VpcVirtualRouter', - "NetworkACL": 'VpcVirtualRouter' - }, - }, - "vpc_offering": { - "name": 'VPC off', - "displaytext": 'VPC off', - "supportedservices": 'Dhcp,Dns,SourceNat,PortForwarding,Vpn,Lb,UserData,StaticNat', - }, - "vpc": { - "name": "TestVPC", - "displaytext": "TestVPC", - "cidr": '10.0.0.1/24' - }, - "network": { - "name": "Test Network", - "displaytext": "Test Network", - "netmask": '255.255.255.0' - }, - "lbrule": { - "name": "SSH", - "alg": "leastconn", - # Algorithm used for load balancing - "privateport": 22, - "publicport": 2222, - "openfirewall": False, - "startport": 22, - "endport": 2222, - "protocol": "TCP", - "cidrlist": '0.0.0.0/0', - }, - "lbrule_http": { - "name": "HTTP", - "alg": "leastconn", - # Algorithm used for load balancing - "privateport": 80, - "publicport": 8888, - "openfirewall": False, - "startport": 80, - "endport": 8888, - "protocol": "TCP", - "cidrlist": '0.0.0.0/0', - }, - "natrule": { - "privateport": 22, - "publicport": 22, - "startport": 22, - "endport": 22, - "protocol": "TCP", - "cidrlist": '0.0.0.0/0', - }, - "http_rule": { - "privateport": 80, - "publicport": 80, - "startport": 80, - "endport": 80, - "cidrlist": '0.0.0.0/0', - "protocol": "TCP" - }, - "virtual_machine": { - "displayname": "Test VM", - "username": "root", - "password": "password", - "ssh_port": 22, - "hypervisor": 'XenServer', - # Hypervisor type should be same as - # hypervisor type of cluster - "privateport": 22, - "publicport": 22, - "protocol": 'TCP', - }, - "ostype": 'CentOS 5.3 (64-bit)', - "sleep": 60, - "timeout": 10, - "mode": 'advanced' - } + "account": { + "email": "test@test.com", + "firstname": "Test", + "lastname": "User", + "username": "test", + # Random characters are appended for unique + # username + "password": "password", + }, + "host1": None, + "host2": None, + "service_offering": { + "name": "Tiny Instance", + "displaytext": "Tiny Instance", + "cpunumber": 1, + "cpuspeed": 1000, + "memory": 512, + }, + "network_offering": { + "name": 'VPC Network offering', + "displaytext": 'VPC Network off', + "guestiptype": 'Isolated', + "supportedservices": 'Vpn,Dhcp,Dns,SourceNat,PortForwarding,Lb,UserData,StaticNat,NetworkACL', + "traffictype": 'GUEST', + "availability": 'Optional', + "useVpc": 'on', + "serviceProviderList": { + "Vpn": 'VpcVirtualRouter', + "Dhcp": 'VpcVirtualRouter', + "Dns": 'VpcVirtualRouter', + "SourceNat": 'VpcVirtualRouter', + "PortForwarding": 'VpcVirtualRouter', + "Lb": 'VpcVirtualRouter', + "UserData": 'VpcVirtualRouter', + "StaticNat": 'VpcVirtualRouter', + "NetworkACL": 'VpcVirtualRouter' + }, + "servicecapabilitylist": { + }, + }, + "network_offering_no_lb": { + "name": 'VPC Network offering', + "displaytext": 'VPC Network off', + "guestiptype": 'Isolated', + "supportedservices": 'Dhcp,Dns,SourceNat,PortForwarding,UserData,StaticNat,NetworkACL', + "traffictype": 'GUEST', + "availability": 'Optional', + "useVpc": 'on', + "serviceProviderList": { + "Dhcp": 'VpcVirtualRouter', + "Dns": 'VpcVirtualRouter', + "SourceNat": 'VpcVirtualRouter', + "PortForwarding": 'VpcVirtualRouter', + "UserData": 'VpcVirtualRouter', + "StaticNat": 'VpcVirtualRouter', + "NetworkACL": 'VpcVirtualRouter' + }, + }, + "vpc_offering": { + "name": 'VPC off', + "displaytext": 'VPC off', + "supportedservices": 'Dhcp,Dns,SourceNat,PortForwarding,Vpn,Lb,UserData,StaticNat', + }, + "vpc": { + "name": "TestVPC", + "displaytext": "TestVPC", + "cidr": '10.0.0.1/24' + }, + "network": { + "name": "Test Network", + "displaytext": "Test Network", + "netmask": '255.255.255.0' + }, + "lbrule": { + "name": "SSH", + "alg": "leastconn", + # Algorithm used for load balancing + "privateport": 22, + "publicport": 2222, + "openfirewall": False, + "startport": 22, + "endport": 2222, + "protocol": "TCP", + "cidrlist": '0.0.0.0/0', + }, + "lbrule_http": { + "name": "HTTP", + "alg": "leastconn", + # Algorithm used for load balancing + "privateport": 80, + "publicport": 8888, + "openfirewall": False, + "startport": 80, + "endport": 8888, + "protocol": "TCP", + "cidrlist": '0.0.0.0/0', + }, + "natrule": { + "privateport": 22, + "publicport": 22, + "startport": 22, + "endport": 22, + "protocol": "TCP", + "cidrlist": '0.0.0.0/0', + }, + "http_rule": { + "privateport": 80, + "publicport": 80, + "startport": 80, + "endport": 80, + "cidrlist": '0.0.0.0/0', + "protocol": "TCP" + }, + "virtual_machine": { + "displayname": "Test VM", + "username": "root", + "password": "password", + "ssh_port": 22, + "hypervisor": 'XenServer', + # Hypervisor type should be same as + # hypervisor type of cluster + "privateport": 22, + "publicport": 22, + "protocol": 'TCP', + }, + "ostype": 'CentOS 5.3 (64-bit)', + "sleep": 60, + "timeout": 10, + "mode": 'advanced' + } class TestVPCNetworkPFRules(cloudstackTestCase): @@ -251,15 +252,12 @@ class TestVPCNetworkPFRules(cloudstackTestCase): try: #Clean up, terminate the created network offerings cleanup_resources(self.apiclient, self._cleanup) - wait_for_cleanup(self.apiclient, [ - "network.gc.interval", - "network.gc.wait"]) except Exception as e: self.debug("Warning: Exception during cleanup : %s" % e) #raise Exception("Warning: Exception during cleanup : %s" % e) return - def get_Router_For_VPC(self): + def get_vpcrouter(self): routers = list_routers(self.apiclient, account=self.account.name, domainid=self.account.domainid, @@ -276,8 +274,8 @@ class TestVPCNetworkPFRules(cloudstackTestCase): return router - def stop_VPC_VRouter(self): - router = self.get_Router_For_VPC() + def stop_vpcrouter(self): + router = self.get_vpcrouter() self.debug("Stopping router ID: %s" % router.id) cmd = stopRouter.stopRouterCmd() cmd.id = router.id @@ -298,7 +296,7 @@ class TestVPCNetworkPFRules(cloudstackTestCase): ) return router - def start_VPC_VRouter(self, router): + def start_vpcrouter(self, router): # Start the VPC Router cmd = startRouter.startRouterCmd() cmd.id = router.id @@ -348,7 +346,7 @@ class TestVPCNetworkPFRules(cloudstackTestCase): else: self.debug("Failed to wget from VM=%s http server on public_ip=%s" % (vm.name, public_ip.ipaddress.ipaddress)) - def create_StaticNatRule_For_VM(self, vm, public_ip, network): + def create_staticnat(self, vm, public_ip, network): self.debug("Enabling static NAT for IP: %s" % public_ip.ipaddress.ipaddress) try: @@ -364,7 +362,7 @@ class TestVPCNetworkPFRules(cloudstackTestCase): self.fail("Failed to enable static NAT on IP: %s - %s" % ( public_ip.ipaddress.ipaddress, e)) - def create_NatRule_For_VM(self, vm, public_ip, network, services=None): + def create_natrule(self, vm, public_ip, network, services=None): self.debug("Creatinng NAT rule in network for vm with public IP") if not services: services = self.services["natrule"] @@ -386,7 +384,7 @@ class TestVPCNetworkPFRules(cloudstackTestCase): self.debug('nwacl_nat=%s' % nwacl_nat.__dict__) return nat_rule - def acquire_Public_IP(self, network): + def acquire_publicip(self, network): self.debug("Associating public IP for network: %s" % network.name) public_ip = PublicIPAddress.create(self.apiclient, accountid=self.account.name, @@ -400,7 +398,7 @@ class TestVPCNetworkPFRules(cloudstackTestCase): )) return public_ip - def create_VPC(self, cidr='10.1.2.1/16'): + def create_vpc(self, cidr='10.1.2.1/16'): self.debug("Creating a VPC offering..") self.services["vpc_offering"]["name"] = self.services["vpc_offering"]["name"] + str(cidr) vpc_off = VpcOffering.create( @@ -424,7 +422,7 @@ class TestVPCNetworkPFRules(cloudstackTestCase): ) return vpc - def create_Network(self, net_offerring, gateway='10.1.1.1',vpc=None): + def create_network(self, net_offerring, gateway='10.1.1.1',vpc=None): try: self.debug('Create NetworkOffering') net_offerring["name"] = "NET_OFF-" + str(gateway) @@ -453,7 +451,7 @@ class TestVPCNetworkPFRules(cloudstackTestCase): except: self.fail('Unable to create a Network with offering=%s' % net_offerring) - def create_VM_in_Network(self, network, host_id=None): + def deployvm_in_network(self, network, host_id=None): try: self.debug('Creating VM in network=%s' % network.name) vm = VirtualMachine.create( @@ -471,7 +469,7 @@ class TestVPCNetworkPFRules(cloudstackTestCase): except: self.fail('Unable to create VM in a Network=%s' % network.name) - def create_LB_Rule(self, public_ip, network, vmarray, services=None): + def create_lbrule(self, public_ip, network, vmarray, services=None): self.debug("Creating LB rule for IP address: %s" % public_ip.ipaddress.ipaddress) objservices = None @@ -493,7 +491,7 @@ class TestVPCNetworkPFRules(cloudstackTestCase): lb_rule.assign(self.apiclient, vmarray) return lb_rule - def create_egress_Internet_Rule(self, network): + def open_egress_to_world(self, network): self.debug("Adding Egress rules to network %s and %s to allow access to internet" % (network.name,self.services["http_rule"])) nwacl_internet_1 = NetworkACL.create( self.apiclient, @@ -507,8 +505,7 @@ class TestVPCNetworkPFRules(cloudstackTestCase): @attr(tags=["advanced", "intervlan"]) def test_01_network_services_VPC_StopCreatePF(self): - """ Test case no 204 : Create PF rules for a single virtual network of a VPC, - using a new Public IP Address available with the VPC when Virtual Router is in Stopped State + """ Test : Create VPC PF rules on acquired public ip when VpcVirtualRouter is stopped """ # Validate the following @@ -521,20 +518,19 @@ class TestVPCNetworkPFRules(cloudstackTestCase): # 7. Start VPC Virtual Router. # 8. Successfully ssh into the Guest VM using the PF rule - network_1 = self.create_Network(self.services["network_offering"]) - vm_1 = self.create_VM_in_Network(network_1) - public_ip_1 = self.acquire_Public_IP(network_1) - router = self.stop_VPC_VRouter() - self.create_NatRule_For_VM( vm_1, public_ip_1, network_1) - self.start_VPC_VRouter(router) + network_1 = self.create_network(self.services["network_offering"]) + vm_1 = self.deployvm_in_network(network_1) + public_ip_1 = self.acquire_publicip(network_1) + router = self.stop_vpcrouter() + self.create_natrule( vm_1, public_ip_1, network_1) + self.start_vpcrouter(router) self.check_ssh_into_vm(vm_1, public_ip_1, testnegative=False) return @attr(tags=["advanced", "intervlan"]) def test_02_network_services_VPC_CreatePF(self): - """ Test case no 190 : Create PF rules for a single virtual network of a VPC using a - new Public IP Address available with the VPC when Virtual Router is in Running State + """ Test Create VPC PF rules on acquired public ip when VpcVirtualRouter is Running """ # Validate the following @@ -545,17 +541,16 @@ class TestVPCNetworkPFRules(cloudstackTestCase): # 5. Use the Create PF rule for vm in network1. # 6. Successfully ssh into the Guest VM using the PF rule - network_1 = self.create_Network(self.services["network_offering"]) - vm_1 = self.create_VM_in_Network(network_1) - public_ip_1 = self.acquire_Public_IP(network_1) - self.create_NatRule_For_VM( vm_1, public_ip_1, network_1) + network_1 = self.create_network(self.services["network_offering"]) + vm_1 = self.deployvm_in_network(network_1) + public_ip_1 = self.acquire_publicip(network_1) + self.create_natrule( vm_1, public_ip_1, network_1) self.check_ssh_into_vm(vm_1, public_ip_1, testnegative=False) return @attr(tags=["advanced", "intervlan"]) def test_03_network_services_VPC_StopCreateMultiplePF(self): - """ Test case no 205 : Create PF rules for a two/multiple virtual networks of a VPC using - a new Public IP Address available with the VPC when Virtual Router is in Stopped State + """ Test Create multiple VPC PF rules on acquired public ip in diff't networks when VpcVirtualRouter is stopped """ # Validate the following @@ -571,24 +566,23 @@ class TestVPCNetworkPFRules(cloudstackTestCase): # 10. Start VPC Virtual Router. # 11. Successfully ssh into the Guest VM1 and VM2 using the PF rule - network_1 = self.create_Network(self.services["network_offering_no_lb"]) - network_2 = self.create_Network(self.services["network_offering_no_lb"], '10.1.2.1') - vm_1 = self.create_VM_in_Network(network_1) - vm_2 = self.create_VM_in_Network(network_2) - public_ip_1 = self.acquire_Public_IP(network_1) - public_ip_2 = self.acquire_Public_IP(network_2) - router = self.stop_VPC_VRouter() - self.create_NatRule_For_VM(vm_1, public_ip_1, network_1) - self.create_NatRule_For_VM(vm_2, public_ip_2, network_2) - self.start_VPC_VRouter(router) + network_1 = self.create_network(self.services["network_offering_no_lb"]) + network_2 = self.create_network(self.services["network_offering_no_lb"], '10.1.2.1') + vm_1 = self.deployvm_in_network(network_1) + vm_2 = self.deployvm_in_network(network_2) + public_ip_1 = self.acquire_publicip(network_1) + public_ip_2 = self.acquire_publicip(network_2) + router = self.stop_vpcrouter() + self.create_natrule(vm_1, public_ip_1, network_1) + self.create_natrule(vm_2, public_ip_2, network_2) + self.start_vpcrouter(router) self.check_ssh_into_vm(vm_1, public_ip_1, testnegative=False) self.check_ssh_into_vm(vm_2, public_ip_2, testnegative=False) return @attr(tags=["advanced", "intervlan"]) def test_04_network_services_VPC_CreateMultiplePF(self): - """ Test case no 191 : Create PF rules for a two/multiple virtual networks of a VPC using a - new Public IP Address available with the VPC when Virtual Router is in Running State + """ Test Create multiple VPC PF rules on acquired public ip in diff't networks when VpcVirtualRouter is running """ # Validate the following @@ -603,24 +597,23 @@ class TestVPCNetworkPFRules(cloudstackTestCase): # 9. Start VPC Virtual Router. # 10. Successfully ssh into the Guest VM1 and VM2 using the PF rule - network_1 = self.create_Network(self.services["network_offering"]) - network_2 = self.create_Network(self.services["network_offering_no_lb"], '10.1.2.1') - vm_1 = self.create_VM_in_Network(network_1) - vm_2 = self.create_VM_in_Network(network_2) - public_ip_1 = self.acquire_Public_IP(network_1) - public_ip_2 = self.acquire_Public_IP(network_2) - router = self.stop_VPC_VRouter() - self.create_NatRule_For_VM(vm_1, public_ip_1, network_1) - self.create_NatRule_For_VM(vm_2, public_ip_2, network_2) - self.start_VPC_VRouter(router) + network_1 = self.create_network(self.services["network_offering"]) + network_2 = self.create_network(self.services["network_offering_no_lb"], '10.1.2.1') + vm_1 = self.deployvm_in_network(network_1) + vm_2 = self.deployvm_in_network(network_2) + public_ip_1 = self.acquire_publicip(network_1) + public_ip_2 = self.acquire_publicip(network_2) + router = self.stop_vpcrouter() + self.create_natrule(vm_1, public_ip_1, network_1) + self.create_natrule(vm_2, public_ip_2, network_2) + self.start_vpcrouter(router) self.check_ssh_into_vm(vm_1, public_ip_1, testnegative=False) self.check_ssh_into_vm(vm_2, public_ip_2, testnegative=False) return @attr(tags=["advanced", "intervlan"]) def test_05_network_services_VPC_StopDeletePF(self): - """ Test case no 207 : Delete few(not all) PF rules for a single virtual network of - a VPC belonging to a single Public IP Address when Virtual Router is in Stopped State + """ Test delete a PF rule in VPC when VpcVirtualRouter is Stopped """ # Validate the following @@ -636,24 +629,23 @@ class TestVPCNetworkPFRules(cloudstackTestCase): # 10. Start VPC Virtual Router. # 11. wget a file present on http server of VM1 should fail - network_1 = self.create_Network(self.services["network_offering"]) - vm_1 = self.create_VM_in_Network(network_1) - public_ip_1 = self.acquire_Public_IP(network_1) - self.create_NatRule_For_VM(vm_1, public_ip_1, network_1) - http_rule = self.create_NatRule_For_VM(vm_1, public_ip_1, network_1, self.services["http_rule"]) + network_1 = self.create_network(self.services["network_offering"]) + vm_1 = self.deployvm_in_network(network_1) + public_ip_1 = self.acquire_publicip(network_1) + self.create_natrule(vm_1, public_ip_1, network_1) + http_rule = self.create_natrule(vm_1, public_ip_1, network_1, self.services["http_rule"]) #http_rule = self.create_egress_Internet_Rule(network_1) self.check_ssh_into_vm(vm_1, public_ip_1, testnegative=False) self.check_wget_from_vm(vm_1, public_ip_1, testnegative=False) - router = self.stop_VPC_VRouter() + router = self.stop_vpcrouter() http_rule.delete() - self.start_VPC_VRouter(router) + self.start_vpcrouter(router) self.check_wget_from_vm(vm_1, public_ip_1, testnegative=True) return @attr(tags=["advanced", "intervlan"]) def test_06_network_services_VPC_DeletePF(self): - """ Test case no 193 : Delete few(not all) PF rules for a single virtual network of - a VPC belonging to a single Public IP Address when Virtual Router is in Running State + """ Test delete a PF rule in VPC when VpcVirtualRouter is Running """ # Validate the following @@ -667,11 +659,11 @@ class TestVPCNetworkPFRules(cloudstackTestCase): # 9. Delete internet PF rule # 10. wget a file present on http server of VM1 should fail - network_1 = self.create_Network(self.services["network_offering"]) - vm_1 = self.create_VM_in_Network(network_1) - public_ip_1 = self.acquire_Public_IP(network_1) - self.create_NatRule_For_VM(vm_1, public_ip_1, network_1) - http_rule=self.create_NatRule_For_VM(vm_1, public_ip_1, network_1, self.services["http_rule"]) + network_1 = self.create_network(self.services["network_offering"]) + vm_1 = self.deployvm_in_network(network_1) + public_ip_1 = self.acquire_publicip(network_1) + self.create_natrule(vm_1, public_ip_1, network_1) + http_rule=self.create_natrule(vm_1, public_ip_1, network_1, self.services["http_rule"]) #http_rule = self.create_egress_Internet_Rule(network_1) self.check_ssh_into_vm(vm_1, public_ip_1, testnegative=False) self.check_wget_from_vm(vm_1, public_ip_1, testnegative=False) @@ -681,8 +673,7 @@ class TestVPCNetworkPFRules(cloudstackTestCase): @attr(tags=["advanced", "intervlan"]) def test_07_network_services_VPC_StopDeleteAllPF(self): - """ Test case no 208 : Delete all PF rules for a single virtual network of a - VPC belonging to a single Public IP Address when Virtual Router is in Stopped State + """ Test delete all PF rules in VPC when VpcVirtualRouter is Stopped """ # Validate the following @@ -699,26 +690,25 @@ class TestVPCNetworkPFRules(cloudstackTestCase): # 11. wget a file present on http server of VM1 should fail # 12. ssh into Guest VM using the PF rule should fail - network_1 = self.create_Network(self.services["network_offering"]) - vm_1 = self.create_VM_in_Network(network_1) - public_ip_1 = self.acquire_Public_IP(network_1) - nat_rule = self.create_NatRule_For_VM(vm_1, public_ip_1, network_1) - http_rule = self.create_NatRule_For_VM(vm_1, public_ip_1, network_1, self.services["http_rule"]) + network_1 = self.create_network(self.services["network_offering"]) + vm_1 = self.deployvm_in_network(network_1) + public_ip_1 = self.acquire_publicip(network_1) + nat_rule = self.create_natrule(vm_1, public_ip_1, network_1) + http_rule = self.create_natrule(vm_1, public_ip_1, network_1, self.services["http_rule"]) #http_rule = self.create_egress_Internet_Rule(network_1) self.check_ssh_into_vm(vm_1, public_ip_1, testnegative=False) self.check_wget_from_vm(vm_1, public_ip_1, testnegative=False) - router = self.stop_VPC_VRouter() + router = self.stop_vpcrouter() http_rule.delete() nat_rule.delete() - self.start_VPC_VRouter(router) + self.start_vpcrouter(router) self.check_ssh_into_vm(vm_1, public_ip_1, testnegative=True) self.check_wget_from_vm(vm_1, public_ip_1, testnegative=True) return @attr(tags=["advanced", "intervlan"]) def test_08_network_services_VPC_DeleteAllPF(self): - """ Test case no 194 : Delete all PF rules for a single virtual network of a - VPC belonging to a single Public IP Address when Virtual Router is in Running State + """ Test delete all PF rules in VPC when VpcVirtualRouter is Running """ # Validate the following @@ -733,11 +723,11 @@ class TestVPCNetworkPFRules(cloudstackTestCase): # 9. wget a file present on http server of VM1 should fail # 10. ssh into Guest VM using the PF rule should fail - network_1 = self.create_Network(self.services["network_offering"]) - vm_1 = self.create_VM_in_Network(network_1) - public_ip_1 = self.acquire_Public_IP(network_1) - nat_rule = self.create_NatRule_For_VM(vm_1, public_ip_1, network_1) - http_rule = self.create_NatRule_For_VM(vm_1, public_ip_1, network_1, self.services["http_rule"]) + network_1 = self.create_network(self.services["network_offering"]) + vm_1 = self.deployvm_in_network(network_1) + public_ip_1 = self.acquire_publicip(network_1) + nat_rule = self.create_natrule(vm_1, public_ip_1, network_1) + http_rule = self.create_natrule(vm_1, public_ip_1, network_1, self.services["http_rule"]) #http_rule = self.create_egress_Internet_Rule(network_1) self.check_ssh_into_vm(vm_1, public_ip_1, testnegative=False) self.check_wget_from_vm(vm_1, public_ip_1, testnegative=False) @@ -749,8 +739,7 @@ class TestVPCNetworkPFRules(cloudstackTestCase): @attr(tags=["advanced", "intervlan"]) def test_09_network_services_VPC_StopDeleteAllMultiplePF(self): - """ Test case no 209 : Delete all PF rules for two/multiple virtual networks of a VPC. - Observe the status of the Public IP Addresses of the rules when Virtual Router is in Stopped State + """ Test delete all PF rules in VPC across multiple networks when VpcVirtualRouter is Stopped """ # Validate the following @@ -769,20 +758,20 @@ class TestVPCNetworkPFRules(cloudstackTestCase): # 12. Start VPC Virtual Router. # 13. Fail to ssh and http to vm1, vm2, vm3 and vm4. - network_1 = self.create_Network(self.services["network_offering"]) - network_2 = self.create_Network(self.services["network_offering_no_lb"], '10.1.2.1') - vm_1 = self.create_VM_in_Network(network_1) - vm_2 = self.create_VM_in_Network(network_1) - vm_3 = self.create_VM_in_Network(network_2) - vm_4 = self.create_VM_in_Network(network_2) - public_ip_1 = self.acquire_Public_IP(network_1) - public_ip_2 = self.acquire_Public_IP(network_1) - nat_rule1 = self.create_NatRule_For_VM(vm_1, public_ip_1, network_1) - nat_rule2 = self.create_NatRule_For_VM(vm_2, public_ip_2, network_1) - http_rule1 = self.create_egress_Internet_Rule(network_1) - nat_rule3 = self.create_NatRule_For_VM(vm_3, public_ip_1, network_2) - nat_rule4 = self.create_NatRule_For_VM(vm_4, public_ip_2, network_2) - http_rule2 = self.create_egress_Internet_Rule(network_2) + network_1 = self.create_network(self.services["network_offering"]) + network_2 = self.create_network(self.services["network_offering_no_lb"], '10.1.2.1') + vm_1 = self.deployvm_in_network(network_1) + vm_2 = self.deployvm_in_network(network_1) + vm_3 = self.deployvm_in_network(network_2) + vm_4 = self.deployvm_in_network(network_2) + public_ip_1 = self.acquire_publicip(network_1) + public_ip_2 = self.acquire_publicip(network_1) + nat_rule1 = self.create_natrule(vm_1, public_ip_1, network_1) + nat_rule2 = self.create_natrule(vm_2, public_ip_2, network_1) + http_rule1 = self.open_egress_to_world(network_1) + nat_rule3 = self.create_natrule(vm_3, public_ip_1, network_2) + nat_rule4 = self.create_natrule(vm_4, public_ip_2, network_2) + http_rule2 = self.open_egress_to_world(network_2) self.check_ssh_into_vm(vm_1, public_ip_1, testnegative=False) self.check_ssh_into_vm(vm_2, public_ip_2, testnegative=False) self.check_ssh_into_vm(vm_3, public_ip_1, testnegative=False) @@ -791,14 +780,14 @@ class TestVPCNetworkPFRules(cloudstackTestCase): self.check_wget_from_vm(vm_2, public_ip_2, testnegative=False) self.check_wget_from_vm(vm_3, public_ip_1, testnegative=False) self.check_wget_from_vm(vm_4, public_ip_2, testnegative=False) - router = self.stop_VPC_VRouter() + router = self.stop_vpcrouter() nat_rule1.delete() nat_rule2.delete() nat_rule3.delete() nat_rule4.delete() http_rule1.delete() http_rule2.delete() - self.start_VPC_VRouter(router) + self.start_vpcrouter(router) self.check_ssh_into_vm(vm_1, public_ip_1, testnegative=True) self.check_ssh_into_vm(vm_2, public_ip_2, testnegative=True) self.check_ssh_into_vm(vm_3, public_ip_1, testnegative=True) @@ -811,10 +800,8 @@ class TestVPCNetworkPFRules(cloudstackTestCase): @attr(tags=["advanced", "intervlan"]) def test_10_network_services_VPC_DeleteAllMultiplePF(self): - """ Test case no 195: Delete all PF rules for two/multiple virtual networks of a VPC. - Observe the status of the Public IP Addresses of the rules when Virtual Router is in Running State + """ Test delete all PF rules in VPC across multiple networks when VpcVirtualRouter is Running """ - # Validate the following # 1. Create a VPC with cidr - 10.1.1.1/16. # 2. Create a Network offering - NO1 with all supported services. @@ -829,20 +816,20 @@ class TestVPCNetworkPFRules(cloudstackTestCase): # 12. Delete all PF rultes for vm1, vm2, vm3 and vm4. # 13. Fail to ssh and http to vm1, vm2, vm3 and vm4. - network_1 = self.create_Network(self.services["network_offering"]) - network_2 = self.create_Network(self.services["network_offering_no_lb"], '10.1.2.1') - vm_1 = self.create_VM_in_Network(network_1) - vm_2 = self.create_VM_in_Network(network_1) - vm_3 = self.create_VM_in_Network(network_2) - vm_4 = self.create_VM_in_Network(network_2) - public_ip_1 = self.acquire_Public_IP(network_1) - public_ip_2 = self.acquire_Public_IP(network_1) - nat_rule1 = self.create_NatRule_For_VM(vm_1, public_ip_1, network_1) - nat_rule2 = self.create_NatRule_For_VM(vm_2, public_ip_2, network_1) - http_rule1 = self.create_egress_Internet_Rule(network_1) - nat_rule3 = self.create_NatRule_For_VM(vm_3, public_ip_1, network_2) - nat_rule4 = self.create_NatRule_For_VM(vm_4, public_ip_2, network_2) - http_rule2 = self.create_egress_Internet_Rule(network_2) + network_1 = self.create_network(self.services["network_offering"]) + network_2 = self.create_network(self.services["network_offering_no_lb"], '10.1.2.1') + vm_1 = self.deployvm_in_network(network_1) + vm_2 = self.deployvm_in_network(network_1) + vm_3 = self.deployvm_in_network(network_2) + vm_4 = self.deployvm_in_network(network_2) + public_ip_1 = self.acquire_publicip(network_1) + public_ip_2 = self.acquire_publicip(network_1) + nat_rule1 = self.create_natrule(vm_1, public_ip_1, network_1) + nat_rule2 = self.create_natrule(vm_2, public_ip_2, network_1) + http_rule1 = self.open_egress_to_world(network_1) + nat_rule3 = self.create_natrule(vm_3, public_ip_1, network_2) + nat_rule4 = self.create_natrule(vm_4, public_ip_2, network_2) + http_rule2 = self.open_egress_to_world(network_2) self.check_ssh_into_vm(vm_1, public_ip_1, testnegative=False) self.check_ssh_into_vm(vm_2, public_ip_2, testnegative=False) self.check_ssh_into_vm(vm_3, public_ip_1, testnegative=False) From d8a235ee316af7d9b21ea62581c8b5c8925081d8 Mon Sep 17 00:00:00 2001 From: Nitin Mehta Date: Fri, 23 Nov 2012 14:11:51 +0530 Subject: [PATCH 26/30] CLOUDSTACK-2923: Delete Secondary storage of a Zone was giving NPE bcz we were still refering to the object. Instead log that the cpvm and ssvm cant be created bcz sec storage is not available --- .../com/cloud/consoleproxy/ConsoleProxyManagerImpl.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java b/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java index 421e53f75be..7362cf1e227 100755 --- a/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java +++ b/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java @@ -945,6 +945,13 @@ public class ConsoleProxyManagerImpl extends ManagerBase implements ConsoleProxy VMTemplateVO template = _templateDao.findSystemVMTemplate(dataCenterId); HostVO secondaryStorageHost = this.templateMgr.getSecondaryStorageHost(dataCenterId); boolean templateReady = false; + + if (secondaryStorageHost == null) { + if (s_logger.isDebugEnabled()) { + s_logger.debug("No secondary storage available in zone " + dataCenterId + ", wait until it is ready to launch secondary storage vm"); + } + return false; + } if (template != null && secondaryStorageHost != null) { VMTemplateHostVO templateHostRef = _vmTemplateHostDao.findByHostTemplate(secondaryStorageHost.getId(), template.getId()); From 8220b9867c70166705fc4ed6373ea8266e754678 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Mon, 10 Jun 2013 14:00:05 +0200 Subject: [PATCH 27/30] destroy vm after test in test_advancedsg_networks.py --- test/integration/component/test_advancedsg_networks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/integration/component/test_advancedsg_networks.py b/test/integration/component/test_advancedsg_networks.py index e24254d4b90..229cfd6e5f9 100644 --- a/test/integration/component/test_advancedsg_networks.py +++ b/test/integration/component/test_advancedsg_networks.py @@ -730,6 +730,7 @@ class TestNetworksInAdvancedSG(cloudstackTestCase): networkids=self.shared_network_sg.id, serviceofferingid=self.service_offering.id ) + self.cleanup_vms.append(self.shared_network_admin_account_virtual_machine) vms = VirtualMachine.list( self.api_client, id=self.shared_network_admin_account_virtual_machine.id, From 78811c50021202cb2374ceeb9edf0fdac5c66398 Mon Sep 17 00:00:00 2001 From: radhikap Date: Fri, 7 Jun 2013 16:44:02 +0530 Subject: [PATCH 28/30] CLOUDSTACK-2404 more conceptual info --- docs/en-US/pvlan.xml | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/docs/en-US/pvlan.xml b/docs/en-US/pvlan.xml index 96c1a78a85d..5084ec411da 100644 --- a/docs/en-US/pvlan.xml +++ b/docs/en-US/pvlan.xml @@ -22,15 +22,37 @@
Isolation in Advanced Zone Using Private VLAN + + + isolate VMs from other VMs on the same network (Shared Networks are the most common use + case) using PVLANs + + + create a Network Offering enabling PVLAN support + + + create shared networks based on a network offering which has PVLANs enabled + + + supported in VPC as well as non-VPC deployments + + + supported on all Hypervisors + + + Allow end users to deploy VMs on Isolated Networks or VPC along with the Shared Networks + that have PVLAN support + +
About Private VLAN - In an Ethernet switch, a VLAN is a broadcast domain in which hosts can establish direct + In an Ethernet switch, a VLAN is a broadcast domain where hosts can establish direct communication with each another at Layer 2. Private VLAN is designed as an extension of VLAN standard to add further segmentation of the logical broadcast domain. A regular VLAN is a single broadcast domain, whereas a private VLAN partitions a larger VLAN broadcast domain into smaller sub-domains. A sub-domain is represented by a pair of VLANs: a Primary VLAN and a - Secondary VLAN. The original VLAN that is being divided into smaller groups is called - Primary, That implies all VLAN pairs in a private VLAN share the same Primary VLAN. All the + Secondary VLAN. The original VLAN that is being divided into smaller groups is called Primary, + which implies that all VLAN pairs in a private VLAN share the same Primary VLAN. All the secondary VLANs exist only inside the Primary. Each Secondary VLAN has a specific VLAN ID associated to it, which differentiates one sub-domain from another. For further reading: @@ -50,6 +72,9 @@
+
+ Prerequisites +
Prerequisites Ensure that you configure private VLAN on your physical switches out-of-band. From 840e14de0b322013a496f62958889383f9ccc1e3 Mon Sep 17 00:00:00 2001 From: radhikap Date: Mon, 10 Jun 2013 17:56:27 +0530 Subject: [PATCH 29/30] pvlan --- docs/en-US/pvlan.xml | 117 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 99 insertions(+), 18 deletions(-) diff --git a/docs/en-US/pvlan.xml b/docs/en-US/pvlan.xml index 5084ec411da..e3f2ea3ace7 100644 --- a/docs/en-US/pvlan.xml +++ b/docs/en-US/pvlan.xml @@ -21,27 +21,25 @@ -->
Isolation in Advanced Zone Using Private VLAN - + Isolation of guest traffic in shared networks can be achieved by using Private VLANs + (PVLAN). PVLANs provide Layer 2 isolation between ports within the same VLAN. In a PVLAN-enabled + shared network, a user VM cannot reach other user VM though they can reach the DHCP server and + gateway, this would in turn allow users to control traffic within a network and help them deploy + multiple applications without communication between application as well as prevent communication + with other users’ VMs. - isolate VMs from other VMs on the same network (Shared Networks are the most common use - case) using PVLANs + Isolate VMs in a shared networks by using Private VLANs. - create a Network Offering enabling PVLAN support + Supported in both VPC and non-VPC deployments. - create shared networks based on a network offering which has PVLANs enabled + Supported on all hypervisors. - supported in VPC as well as non-VPC deployments - - - supported on all Hypervisors - - - Allow end users to deploy VMs on Isolated Networks or VPC along with the Shared Networks - that have PVLAN support + Allow end users to deploy VMs in an isolated networks, or a VPC, or a Private + VLAN-enabled shared network.
@@ -54,7 +52,38 @@ Secondary VLAN. The original VLAN that is being divided into smaller groups is called Primary, which implies that all VLAN pairs in a private VLAN share the same Primary VLAN. All the secondary VLANs exist only inside the Primary. Each Secondary VLAN has a specific VLAN ID - associated to it, which differentiates one sub-domain from another. + associated to it, which differentiates one sub-domain from another. + Three types of ports exist in a private VLAN domain, which essentially determine the + behaviour of the participating hosts. Each ports will have its own unique set of rules, which + regulate a connected host's ability to communicate with other connected host within the same + private VLAN domain. Configure each host that is part of a PVLAN pair can be by using one of + these three port designation: + + + Promiscuous: A promiscuous port can communicate with + all the interfaces, including the community and isolated host ports that belong to the + secondary VLANs. In Promiscuous mode, hosts are connected to promiscuous ports and are + able to communicate directly with resources on both primary and secondary VLAN. Routers, + DHCP servers, and other trusted devices are typically attached to promiscuous + ports. + + + Isolated VLANs: The ports within an isolated VLAN + cannot communicate with each other at the layer-2 level. The hosts that are connected to + Isolated ports can directly communicate only with the Promiscuous resources. If your + customer device needs to have access only to a gateway router, attach it to an isolated + port. + + + Community VLANs: The ports within a community VLAN + can communicate with each other and with the promiscuous ports, but they cannot + communicate with the ports in other communities at the layer-2 level. In a Community mode, + direct communication is permitted only with the hosts in the same community and those that + are connected to the Primary PVLAN in promiscuous mode. If your customer has two devices + that need to be isolated from other customers' devices, but to be able to communicate + among themselves, deploy them in community ports. + + For further reading: @@ -72,11 +101,63 @@
-
- Prerequisites -
Prerequisites - Ensure that you configure private VLAN on your physical switches out-of-band. + + + Use a PVLAN supported switch. + See Private VLAN Catalyst Switch Support Matrixfor more information. + + + Connect a switch to the gateway; connect additional switches to the gateway via a + trunk port: Only Cisco Catalyst 4500 has the PVLAN promiscuous trunk mode to connect both + normal VLAN and PVLAN to a PVLAN-unaware switch. For other Catalyst PVLAN support switch, + connect the switch to upper switch by using cables. The number of cables should be greater + than the number of PVLANs used. + + + All the layer 2 switches, which are PVLAN-aware, are connected to each other, and one + of them is connected to a router. All the ports connected to the host would be configured + in trunk mode. Allow Management VLAN, Primary VLAN (public) and secondary Isolated VLAN + ports. Configure the switch port connected to the router in PVLAN promiscuous trunk mode, + which would translate an isolated VLAN to primary VLAN for router, which is PVLAN-unaware. + + + + If your Catalyst switch supports PVLAN, but not PVLAN promiscuous trunk mode, perform + the following: + + + Configure one of the switch port as trunk for management network (management + VLAN). + + + For each PVLAN, perform the following: + + + Connect one port of the Catalyst switch to the upper switch. + + + Set the port in the Catalyst Switch in promiscuous mode for one pair of + PVLAN + + + Set the port in upper switch to access mode, and allow only the traffic of + primary VLAN of the PVLAN pair. + + + + + + + Configure private VLAN on your physical switches out-of-band. + + +
+
+ + <para/> </section> </section> From 76ce304411259dc4a29e0e070de829b79b46efe2 Mon Sep 17 00:00:00 2001 From: Wei Zhou <w.zhou@leaseweb.com> Date: Mon, 10 Jun 2013 15:00:08 +0200 Subject: [PATCH 30/30] CLOUDSTACK-2707: use executeBatch instead of persist in Usage Server --- .../src/com/cloud/usage/dao/UsageDao.java | 5 +- .../src/com/cloud/usage/dao/UsageDaoImpl.java | 64 +++++++++++++++++++ .../src/com/cloud/api/ApiResponseHelper.java | 4 +- .../usage/parser/NetworkUsageParser.java | 11 +++- .../cloud/usage/parser/VmDiskUsageParser.java | 43 ++++++++----- 5 files changed, 104 insertions(+), 23 deletions(-) diff --git a/engine/schema/src/com/cloud/usage/dao/UsageDao.java b/engine/schema/src/com/cloud/usage/dao/UsageDao.java index 8a806553112..f571b63ce40 100644 --- a/engine/schema/src/com/cloud/usage/dao/UsageDao.java +++ b/engine/schema/src/com/cloud/usage/dao/UsageDao.java @@ -38,6 +38,7 @@ public interface UsageDao extends GenericDao<UsageVO, Long> { Long getLastUserStatsId(); List<Long> listPublicTemplatesByAccount(long accountId); Long getLastVmDiskStatsId(); - void updateVmDiskStats(List<VmDiskStatisticsVO> vmNetStats); - void saveVmDiskStats(List<VmDiskStatisticsVO> vmNetStats); + void updateVmDiskStats(List<VmDiskStatisticsVO> vmDiskStats); + void saveVmDiskStats(List<VmDiskStatisticsVO> vmDiskStats); + void saveUsageRecords(List<UsageVO> usageRecords); } diff --git a/engine/schema/src/com/cloud/usage/dao/UsageDaoImpl.java b/engine/schema/src/com/cloud/usage/dao/UsageDaoImpl.java index f7d5069eef9..2237d56dd72 100644 --- a/engine/schema/src/com/cloud/usage/dao/UsageDaoImpl.java +++ b/engine/schema/src/com/cloud/usage/dao/UsageDaoImpl.java @@ -18,6 +18,7 @@ package com.cloud.usage.dao; import java.sql.PreparedStatement; import java.sql.ResultSet; +import java.sql.Timestamp; import java.sql.Types; import java.util.ArrayList; import java.util.Date; @@ -63,6 +64,8 @@ public class UsageDaoImpl extends GenericDaoBase<UsageVO, Long> implements Usage " VALUES (?,?,?,?,?,?,?,?,?,?, ?, ?, ?, ?,?, ?, ?)"; private static final String UPDATE_VM_DISK_STATS = "UPDATE cloud_usage.vm_disk_statistics SET net_io_read=?, net_io_write=?, current_io_read=?, current_io_write=?, agg_io_read=?, agg_io_write=?, " + "net_bytes_read=?, net_bytes_write=?, current_bytes_read=?, current_bytes_write=?, agg_bytes_read=?, agg_bytes_write=? WHERE id=?"; + private static final String INSERT_USGAE_RECORDS = "INSERT INTO cloud_usage.cloud_usage (zone_id, account_id, domain_id, description, usage_display, usage_type, raw_usage, vm_instance_id, vm_name, offering_id, template_id, " + + "usage_id, type, size, network_id, start_date, end_date) VALUES (?,?,?,?,?,?,?,?,?, ?, ?, ?,?,?,?,?,?)"; protected final static TimeZone s_gmtTimeZone = TimeZone.getTimeZone("GMT"); @@ -375,4 +378,65 @@ public class UsageDaoImpl extends GenericDaoBase<UsageVO, Long> implements Usage } } + + @Override + public void saveUsageRecords(List<UsageVO> usageRecords) { + Transaction txn = Transaction.currentTxn(); + try { + txn.start(); + String sql = INSERT_USGAE_RECORDS; + PreparedStatement pstmt = null; + pstmt = txn.prepareAutoCloseStatement(sql); // in reality I just want CLOUD_USAGE dataSource connection + for (UsageVO usageRecord : usageRecords) { + pstmt.setLong(1, usageRecord.getZoneId()); + pstmt.setLong(2, usageRecord.getAccountId()); + pstmt.setLong(3, usageRecord.getDomainId()); + pstmt.setString(4, usageRecord.getDescription()); + pstmt.setString(5, usageRecord.getUsageDisplay()); + pstmt.setInt(6, usageRecord.getUsageType()); + pstmt.setDouble(7, usageRecord.getRawUsage()); + if(usageRecord.getVmInstanceId() != null){ + pstmt.setLong(8, usageRecord.getVmInstanceId()); + } else { + pstmt.setNull(8, Types.BIGINT); + } + pstmt.setString(9, usageRecord.getVmName()); + if(usageRecord.getOfferingId() != null){ + pstmt.setLong(10, usageRecord.getOfferingId()); + } else { + pstmt.setNull(10, Types.BIGINT); + } + if(usageRecord.getTemplateId() != null){ + pstmt.setLong(11, usageRecord.getTemplateId()); + } else { + pstmt.setNull(11, Types.BIGINT); + } + if(usageRecord.getUsageId() != null){ + pstmt.setLong(12, usageRecord.getUsageId()); + } else { + pstmt.setNull(12, Types.BIGINT); + } + pstmt.setString(13, usageRecord.getType()); + if(usageRecord.getSize() != null){ + pstmt.setLong(14, usageRecord.getSize()); + } else { + pstmt.setNull(14, Types.BIGINT); + } + if(usageRecord.getNetworkId() != null){ + pstmt.setLong(15, usageRecord.getNetworkId()); + } else { + pstmt.setNull(15, Types.BIGINT); + } + pstmt.setTimestamp(16, new Timestamp(usageRecord.getStartDate().getTime())); + pstmt.setTimestamp(17, new Timestamp(usageRecord.getEndDate().getTime())); + pstmt.addBatch(); + } + pstmt.executeBatch(); + txn.commit(); + } catch (Exception ex) { + txn.rollback(); + s_logger.error("error saving usage records to cloud_usage db", ex); + throw new CloudRuntimeException(ex.getMessage()); + } + } } diff --git a/server/src/com/cloud/api/ApiResponseHelper.java b/server/src/com/cloud/api/ApiResponseHelper.java index bcc1605d8b9..0c98abc9400 100755 --- a/server/src/com/cloud/api/ApiResponseHelper.java +++ b/server/src/com/cloud/api/ApiResponseHelper.java @@ -3404,8 +3404,8 @@ public class ApiResponseHelper implements ResponseGenerator { //Device Type usageRecResponse.setType(usageRecord.getType()); //VM Instance Id - VMInstanceVO vm = _entityMgr.findByIdIncludingRemoved(VMInstanceVO.class, usageRecord.getUsageId().toString()); - usageRecResponse.setUsageId(vm.getUuid()); + VMInstanceVO vm = _entityMgr.findByIdIncludingRemoved(VMInstanceVO.class, usageRecord.getVmInstanceId().toString()); + usageRecResponse.setVirtualMachineId(vm.getUuid()); //Volume ID VolumeVO volume = _entityMgr.findByIdIncludingRemoved(VolumeVO.class, usageRecord.getUsageId().toString()); usageRecResponse.setUsageId(volume.getUuid()); diff --git a/usage/src/com/cloud/usage/parser/NetworkUsageParser.java b/usage/src/com/cloud/usage/parser/NetworkUsageParser.java index 3da6854d11b..f2f24e460c9 100644 --- a/usage/src/com/cloud/usage/parser/NetworkUsageParser.java +++ b/usage/src/com/cloud/usage/parser/NetworkUsageParser.java @@ -16,6 +16,7 @@ // under the License. package com.cloud.usage.parser; +import java.util.ArrayList; import java.util.Date; import java.util.HashMap; import java.util.List; @@ -89,6 +90,7 @@ public static final Logger s_logger = Logger.getLogger(NetworkUsageParser.class. networkUsageByZone.put(key, new NetworkInfo(zoneId, usageNetwork.getHostId(), usageNetwork.getHostType(), usageNetwork.getNetworkId(), bytesSent, bytesReceived)); } + List<UsageVO> usageRecords = new ArrayList<UsageVO>(); for (String key : networkUsageByZone.keySet()) { NetworkInfo networkInfo = networkUsageByZone.get(key); long totalBytesSent = networkInfo.getBytesSent(); @@ -110,7 +112,7 @@ public static final Logger s_logger = Logger.getLogger(NetworkUsageParser.class. } UsageVO usageRecord = new UsageVO(networkInfo.getZoneId(), account.getId(), account.getDomainId(), usageDesc, totalBytesSent + " bytes sent", UsageTypes.NETWORK_BYTES_SENT, new Double(totalBytesSent), hostId, networkInfo.getHostType(), networkInfo.getNetworkId(), startDate, endDate); - m_usageDao.persist(usageRecord); + usageRecords.add(usageRecord); // Create the usage record for bytes received usageDesc = "network bytes received"; @@ -119,7 +121,7 @@ public static final Logger s_logger = Logger.getLogger(NetworkUsageParser.class. } usageRecord = new UsageVO(networkInfo.getZoneId(), account.getId(), account.getDomainId(), usageDesc, totalBytesReceived + " bytes received", UsageTypes.NETWORK_BYTES_RECEIVED, new Double(totalBytesReceived), hostId, networkInfo.getHostType(), networkInfo.getNetworkId(), startDate, endDate); - m_usageDao.persist(usageRecord); + usageRecords.add(usageRecord); } else { // Don't charge anything if there were zero bytes processed if (s_logger.isDebugEnabled()) { @@ -127,6 +129,11 @@ public static final Logger s_logger = Logger.getLogger(NetworkUsageParser.class. } } } + try { + m_usageDao.saveUsageRecords(usageRecords); + } catch (Exception ex) { + s_logger.error("Exception in usage manager", ex); + } return true; } diff --git a/usage/src/com/cloud/usage/parser/VmDiskUsageParser.java b/usage/src/com/cloud/usage/parser/VmDiskUsageParser.java index b8a5f98c99b..74fa214324e 100644 --- a/usage/src/com/cloud/usage/parser/VmDiskUsageParser.java +++ b/usage/src/com/cloud/usage/parser/VmDiskUsageParser.java @@ -16,6 +16,7 @@ // under the License. package com.cloud.usage.parser; +import java.util.ArrayList; import java.util.Date; import java.util.HashMap; import java.util.List; @@ -93,6 +94,7 @@ public static final Logger s_logger = Logger.getLogger(VmDiskUsageParser.class.g vmDiskUsageByZone.put(key, new VmDiskInfo(zoneId, usageVmDisk.getVmId(), usageVmDisk.getVolumeId(), ioRead, ioWrite, bytesRead, bytesWrite)); } + List<UsageVO> usageRecords = new ArrayList<UsageVO>(); for (String key : vmDiskUsageByZone.keySet()) { VmDiskInfo vmDiskInfo = vmDiskUsageByZone.get(key); long ioRead = vmDiskInfo.getIORead(); @@ -107,44 +109,45 @@ public static final Logger s_logger = Logger.getLogger(VmDiskUsageParser.class.g } Long vmId = null; + Long volumeId = null; // Create the usage record for bytes read String usageDesc = "disk bytes read"; - if(vmDiskInfo.getVmId() != 0){ + if ((vmDiskInfo.getVmId() != 0) && (vmDiskInfo.getVolumeId() != 0)){ vmId = vmDiskInfo.getVmId(); - usageDesc += " for Vm: "+vmDiskInfo.getVmId()+" and Volume: "+ vmDiskInfo.getVolumeId(); + volumeId = vmDiskInfo.getVolumeId(); + usageDesc += " for Vm: " + vmId + " and Volume: " + volumeId; } UsageVO usageRecord = new UsageVO(vmDiskInfo.getZoneId(), account.getId(), account.getDomainId(), usageDesc, ioRead + " io read", - UsageTypes.VM_DISK_IO_READ, new Double(ioRead), vmId, "VirtualMachine", vmDiskInfo.getVolumeId(), startDate, endDate); - m_usageDao.persist(usageRecord); + UsageTypes.VM_DISK_IO_READ, new Double(ioRead), vmId, null, null, null, vmDiskInfo.getVolumeId(), startDate, endDate, "VirtualMachine"); + usageRecords.add(usageRecord); // Create the usage record for bytes write usageDesc = "disk bytes write"; - if(vmDiskInfo.getVmId() != 0){ - usageDesc += " for Vm: "+vmDiskInfo.getVmId()+" and Volume: "+ vmDiskInfo.getVolumeId(); + if ((vmDiskInfo.getVmId() != 0) && (vmDiskInfo.getVolumeId() != 0)){ + usageDesc += " for Vm: " + vmId + " and Volume: " + volumeId; } usageRecord = new UsageVO(vmDiskInfo.getZoneId(), account.getId(), account.getDomainId(), usageDesc, ioWrite + " io write", - UsageTypes.VM_DISK_BYTES_WRITE, new Double(ioWrite), vmId, "VirtualMachine", vmDiskInfo.getVolumeId(), startDate, endDate); - m_usageDao.persist(usageRecord); + UsageTypes.VM_DISK_BYTES_WRITE, new Double(ioWrite), vmId, null, null, null, vmDiskInfo.getVolumeId(), startDate, endDate, "VirtualMachine"); + usageRecords.add(usageRecord); // Create the usage record for bytes read usageDesc = "disk bytes read"; - if(vmDiskInfo.getVmId() != 0){ - vmId = vmDiskInfo.getVmId(); - usageDesc += " for Vm: "+vmDiskInfo.getVmId()+" and Volume: "+ vmDiskInfo.getVolumeId(); + if ((vmDiskInfo.getVmId() != 0) && (vmDiskInfo.getVolumeId() != 0)){ + usageDesc += " for Vm: " + vmId + " and Volume: " + volumeId; } usageRecord = new UsageVO(vmDiskInfo.getZoneId(), account.getId(), account.getDomainId(), usageDesc, bytesRead + " bytes read", - UsageTypes.VM_DISK_BYTES_READ, new Double(bytesRead), vmId, "VirtualMachine", vmDiskInfo.getVolumeId(), startDate, endDate); - m_usageDao.persist(usageRecord); + UsageTypes.VM_DISK_BYTES_READ, new Double(bytesRead), vmId, null, null, null, vmDiskInfo.getVolumeId(), startDate, endDate, "VirtualMachine"); + usageRecords.add(usageRecord); // Create the usage record for bytes write usageDesc = "disk bytes write"; - if(vmDiskInfo.getVmId() != 0){ - usageDesc += " for Vm: "+vmDiskInfo.getVmId()+" and Volume: "+ vmDiskInfo.getVolumeId(); + if ((vmDiskInfo.getVmId() != 0) && (vmDiskInfo.getVolumeId() != 0)){ + usageDesc += " for Vm: " + vmId + " and Volume: " + volumeId; } usageRecord = new UsageVO(vmDiskInfo.getZoneId(), account.getId(), account.getDomainId(), usageDesc, bytesWrite + " bytes write", - UsageTypes.VM_DISK_BYTES_WRITE, new Double(bytesWrite), vmId, "VirtualMachine", vmDiskInfo.getVolumeId(), startDate, endDate); - m_usageDao.persist(usageRecord); + UsageTypes.VM_DISK_BYTES_WRITE, new Double(bytesWrite), vmId, null, null, null, vmDiskInfo.getVolumeId(), startDate, endDate, "VirtualMachine"); + usageRecords.add(usageRecord); } else { // Don't charge anything if there were zero bytes processed @@ -154,6 +157,12 @@ public static final Logger s_logger = Logger.getLogger(VmDiskUsageParser.class.g } } + try { + m_usageDao.saveUsageRecords(usageRecords); + } catch (Exception ex) { + s_logger.error("Exception in usage manager", ex); + } + return true; }