From 28f04c8ecc26b67b07d21a2ed66edb66a4fef603 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Wed, 31 Jul 2013 12:40:40 +0530 Subject: [PATCH 001/763] CLOUDSTACK-3969: Upgrade failed from 3.0.3 to 4.2 in KVM host with error 4.2.0 KVM SystemVm template not found though systemvm-kvm-4.2 template is uploaded during 3.0.3 setup This is due to mysql version conflict with '=' and 'LIKE'. Changing to '=' while searching for new system template. --- engine/schema/src/com/cloud/upgrade/dao/Upgrade410to420.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/schema/src/com/cloud/upgrade/dao/Upgrade410to420.java b/engine/schema/src/com/cloud/upgrade/dao/Upgrade410to420.java index 5052573d91a..8960c915c06 100644 --- a/engine/schema/src/com/cloud/upgrade/dao/Upgrade410to420.java +++ b/engine/schema/src/com/cloud/upgrade/dao/Upgrade410to420.java @@ -306,7 +306,7 @@ public class Upgrade410to420 implements DbUpgrade { s_logger.debug("Updating " + hypervisorAndTemplateName.getKey() + " System Vms"); try { //Get 4.2.0 system Vm template Id for corresponding hypervisor - pstmt = conn.prepareStatement("select id from `cloud`.`vm_template` where name like ? and removed is null order by id desc limit 1"); + pstmt = conn.prepareStatement("select id from `cloud`.`vm_template` where name = ? and removed is null order by id desc limit 1"); pstmt.setString(1, hypervisorAndTemplateName.getValue()); rs = pstmt.executeQuery(); if(rs.next()){ From 5c18483d8182ea3da9d8a34778b7f98f72bcd29c Mon Sep 17 00:00:00 2001 From: Bharat Kumar Date: Mon, 29 Jul 2013 15:27:11 +0530 Subject: [PATCH 002/763] Cloudstack-2997 unable to update concurrent_conncetions feild in network_offerin. Signed-off-by: Abhinandan Prateek --- .../cloudstack/api/response/NetworkOfferingResponse.java | 6 ++++++ .../src/com/cloud/offerings/NetworkOfferingVO.java | 7 ++++--- server/src/com/cloud/api/ApiResponseHelper.java | 1 + .../cloud/configuration/ConfigurationManagerImpl.java | 9 +++++---- 4 files changed, 16 insertions(+), 7 deletions(-) diff --git a/api/src/org/apache/cloudstack/api/response/NetworkOfferingResponse.java b/api/src/org/apache/cloudstack/api/response/NetworkOfferingResponse.java index 6b35d7b4eaf..a593ae6bc3f 100644 --- a/api/src/org/apache/cloudstack/api/response/NetworkOfferingResponse.java +++ b/api/src/org/apache/cloudstack/api/response/NetworkOfferingResponse.java @@ -91,6 +91,8 @@ public class NetworkOfferingResponse extends BaseResponse { @SerializedName(ApiConstants.EGRESS_DEFAULT_POLICY) @Param(description="true if network offering supports persistent networks, false otherwise") private Boolean egressDefaultPolicy; + @SerializedName(ApiConstants.MAX_CONNECTIONS) @Param(description = "maximum number of concurrents connections to be handled by lb") + private Integer concurrentConnections; public void setId(String id) { this.id = id; @@ -173,4 +175,8 @@ public class NetworkOfferingResponse extends BaseResponse { this.egressDefaultPolicy = egressDefaultPolicy; } + public void setConcurrentConnections(Integer concurrentConnections) { + this.concurrentConnections = concurrentConnections; + } + } diff --git a/engine/schema/src/com/cloud/offerings/NetworkOfferingVO.java b/engine/schema/src/com/cloud/offerings/NetworkOfferingVO.java index 406d98a5286..1ab463bd64a 100755 --- a/engine/schema/src/com/cloud/offerings/NetworkOfferingVO.java +++ b/engine/schema/src/com/cloud/offerings/NetworkOfferingVO.java @@ -134,7 +134,8 @@ public class NetworkOfferingVO implements NetworkOffering { boolean egressdefaultpolicy; @Column(name = "concurrent_connections") - Integer concurrent_connections; + Integer concurrentConnections; + @Override public String getDisplayText() { @@ -433,10 +434,10 @@ public class NetworkOfferingVO implements NetworkOffering { this.publicLb = publicLb; } public Integer getConcurrentConnections() { - return this.concurrent_connections; + return this.concurrentConnections; } public void setConcurrentConnections(Integer concurrent_connections) { - this.concurrent_connections = concurrent_connections; + this.concurrentConnections = concurrent_connections; } } diff --git a/server/src/com/cloud/api/ApiResponseHelper.java b/server/src/com/cloud/api/ApiResponseHelper.java index 307e6bbf53d..4832d71472f 100755 --- a/server/src/com/cloud/api/ApiResponseHelper.java +++ b/server/src/com/cloud/api/ApiResponseHelper.java @@ -2001,6 +2001,7 @@ public class ApiResponseHelper implements ResponseGenerator { response.setIsPersistent(offering.getIsPersistent()); response.setNetworkRate(ApiDBUtils.getNetworkRate(offering.getId())); response.setEgressDefaultPolicy(offering.getEgressDefaultPolicy()); + response.setConcurrentConnections(offering.getConcurrentConnections()); Long so = null; if (offering.getServiceOfferingId() != null) { so = offering.getServiceOfferingId(); diff --git a/server/src/com/cloud/configuration/ConfigurationManagerImpl.java b/server/src/com/cloud/configuration/ConfigurationManagerImpl.java index 3ec35148a7a..9a21df7b806 100755 --- a/server/src/com/cloud/configuration/ConfigurationManagerImpl.java +++ b/server/src/com/cloud/configuration/ConfigurationManagerImpl.java @@ -3926,7 +3926,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati } validateLoadBalancerServiceCapabilities(lbServiceCapabilityMap); - if (lbServiceCapabilityMap != null && !lbServiceCapabilityMap.isEmpty()) { + if (!serviceProviderMap.containsKey(Service.Lb) && lbServiceCapabilityMap != null && !lbServiceCapabilityMap.isEmpty()) { maxconn = cmd.getMaxconnections(); if (maxconn == null) { maxconn=Integer.parseInt(_configDao.getValue(Config.NetworkLBHaproxyMaxConn.key())); @@ -4718,9 +4718,10 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati offering.setAvailability(availability); } } - - if (maxconn != null) { - offering.setConcurrentConnections(maxconn); + if (_ntwkOffServiceMapDao.areServicesSupportedByNetworkOffering(offering.getId(), Service.Lb)){ + if (maxconn != null) { + offering.setConcurrentConnections(maxconn); + } } if (_networkOfferingDao.update(id, offering)) { From 798c507c7d9a0241fbaaddb3348eaa3313b4f9a3 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Wed, 31 Jul 2013 16:56:08 +0530 Subject: [PATCH 003/763] CLOUDSTACK-2717: listCapabilities API to return allow.public.user.templates configuration parameter value defined at account level. Signed by - Nitin Mehta --- server/src/com/cloud/server/ManagementServerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/com/cloud/server/ManagementServerImpl.java b/server/src/com/cloud/server/ManagementServerImpl.java index 26efa10f85c..a1815537480 100755 --- a/server/src/com/cloud/server/ManagementServerImpl.java +++ b/server/src/com/cloud/server/ManagementServerImpl.java @@ -3204,7 +3204,7 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe long diskOffMaxSize = Long.valueOf(_configDao.getValue(Config.CustomDiskOfferingMaxSize.key())); - String userPublicTemplateEnabled = _configs.get(Config.AllowPublicUserTemplates.key()); + String userPublicTemplateEnabled = _configServer.getConfigValue(Config.AllowPublicUserTemplates.key(), Config.ConfigurationParameterScope.account.toString(), caller.getId()); // add some parameters UI needs to handle API throttling boolean apiLimitEnabled = Boolean.parseBoolean(_configDao.getValue(Config.ApiLimitEnabled.key())); From 805d087d8761299a2df5a6fc01da30c1dc42a368 Mon Sep 17 00:00:00 2001 From: Prasanna Santhanam Date: Wed, 31 Jul 2013 15:18:04 +0530 Subject: [PATCH 004/763] Cleanup the marvin logger It is useful to have the same logger used in the marvinplugin into the testcase. The testcase can then append its name to the logger making the logs specify exactly which test is executing when the logs are generated Signed-off-by: Prasanna Santhanam --- tools/marvin/marvin/marvinPlugin.py | 20 ++++---------------- tools/marvin/marvin/remoteSSHClient.py | 4 ++-- 2 files changed, 6 insertions(+), 18 deletions(-) diff --git a/tools/marvin/marvin/marvinPlugin.py b/tools/marvin/marvin/marvinPlugin.py index 29491094db0..f7b0664856c 100644 --- a/tools/marvin/marvin/marvinPlugin.py +++ b/tools/marvin/marvin/marvinPlugin.py @@ -25,11 +25,6 @@ from nose.plugins.base import Plugin from functools import partial -def testCaseLogger(message, logger=None): - if logger is not None: - logger.debug(message) - - class MarvinPlugin(Plugin): """ Custom plugin for the cloudstackTestCases to be run using nose @@ -69,10 +64,6 @@ class MarvinPlugin(Plugin): self.setClient(deploy.testClient) self.setConfig(deploy.getCfg()) - cfg = nose.config.Config() - cfg.logStream = self.result_stream - cfg.debugLog = self.debug_stream - self.testrunner = nose.core.TextTestRunner(stream=self.result_stream, descriptions=True, verbosity=2, config=config) @@ -133,21 +124,18 @@ class MarvinPlugin(Plugin): def beforeTest(self, test): testname = test.__str__().split()[0] self.testclient.identifier = '-'.join([self.identifier, testname]) + self.logger.name = test.__str__() def _injectClients(self, test): - testcaselogger = logging.getLogger("testclient.testcase.%s" % - test.__name__) self.debug_stream. \ setFormatter(logging. Formatter("%(asctime)s - %(levelname)s - %(name)s" + " - %(message)s")) - - testcaselogger.addHandler(self.debug_stream) - testcaselogger.setLevel(logging.DEBUG) - + setattr(test, "debug", self.logger.debug) + setattr(test, "info", self.logger.info) + setattr(test, "warn", self.logger.warning) setattr(test, "testClient", self.testclient) setattr(test, "config", self.config) - setattr(test, "debug", partial(testCaseLogger, logger=testcaselogger)) if self.testclient.identifier is None: self.testclient.identifier = self.identifier setattr(test, "clstestclient", self.testclient) diff --git a/tools/marvin/marvin/remoteSSHClient.py b/tools/marvin/marvin/remoteSSHClient.py index 1dd1ef1a815..764ba2eb95f 100644 --- a/tools/marvin/marvin/remoteSSHClient.py +++ b/tools/marvin/marvin/remoteSSHClient.py @@ -24,7 +24,7 @@ from contextlib import closing class remoteSSHClient(object): - def __init__(self, host, port, user, passwd, retries=10, delay=5, + def __init__(self, host, port, user, passwd, retries=5, delay=30, log_lvl=logging.INFO, keyPairFileLocation=None): self.host = host self.port = port @@ -69,7 +69,7 @@ class remoteSSHClient(object): + "login to %s on port %s" % (str(host), port)) else: - return + return self.ssh def execute(self, command): stdin, stdout, stderr = self.ssh.exec_command(command) From 7aafd06df6a5c8fffe336e0168de6d5fe22083b3 Mon Sep 17 00:00:00 2001 From: Prasanna Santhanam Date: Wed, 31 Jul 2013 17:16:48 +0530 Subject: [PATCH 005/763] Include SSH retry logic when encountering channel failures. Only on SSHExceptions we attempted retries, but during socket failures, like Network Unreachable we failed the ssh connection immediately. Signed-off-by: Prasanna Santhanam --- tools/marvin/marvin/integration/lib/utils.py | 2 +- tools/marvin/marvin/remoteSSHClient.py | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/tools/marvin/marvin/integration/lib/utils.py b/tools/marvin/marvin/integration/lib/utils.py index 92aee836b96..2da9272c7aa 100644 --- a/tools/marvin/marvin/integration/lib/utils.py +++ b/tools/marvin/marvin/integration/lib/utils.py @@ -109,7 +109,7 @@ def cleanup_resources(api_client, resources): obj.delete(api_client) -def is_server_ssh_ready(ipaddress, port, username, password, retries=5, timeout=20, keyPairFileLocation=None): +def is_server_ssh_ready(ipaddress, port, username, password, retries=5, timeout=30, keyPairFileLocation=None): """Return ssh handle else wait till sshd is running""" try: ssh = remoteSSHClient( diff --git a/tools/marvin/marvin/remoteSSHClient.py b/tools/marvin/marvin/remoteSSHClient.py index 764ba2eb95f..e0ead9315a9 100644 --- a/tools/marvin/marvin/remoteSSHClient.py +++ b/tools/marvin/marvin/remoteSSHClient.py @@ -17,6 +17,7 @@ import paramiko import time +import socket import cloudstackException import contextlib import logging @@ -57,10 +58,10 @@ class remoteSSHClient(object): (str(host), user, keyPairFileLocation)) self.logger.debug("SSH connect: %s@%s with passwd %s" % (user, str(host), passwd)) - except paramiko.SSHException, sshex: + except (paramiko.SSHException, paramiko.ChannelException, socket.error) as se: if retry_count == 0: raise cloudstackException. \ - InvalidParameterException(repr(sshex)) + InvalidParameterException(repr(se)) retry_count = retry_count - 1 time.sleep(delay) except paramiko.AuthenticationException, authEx: @@ -68,8 +69,6 @@ class remoteSSHClient(object): InvalidParameterException("Invalid credentials to " + "login to %s on port %s" % (str(host), port)) - else: - return self.ssh def execute(self, command): stdin, stdout, stderr = self.ssh.exec_command(command) From ba619ec112170b916f69a7466a779196f74bef7c Mon Sep 17 00:00:00 2001 From: Bharat Kumar Date: Fri, 26 Jul 2013 12:18:12 +0530 Subject: [PATCH 006/763] recalculate the capacity after overcomit change. --- .../cloud/capacity/CapacityManagerImpl.java | 40 ++++++++++++++++--- .../cloud/vm/VirtualMachineManagerImpl.java | 21 ++++++++++ 2 files changed, 55 insertions(+), 6 deletions(-) diff --git a/server/src/com/cloud/capacity/CapacityManagerImpl.java b/server/src/com/cloud/capacity/CapacityManagerImpl.java index 56c3b71c5ab..cf7f3c4e6ea 100755 --- a/server/src/com/cloud/capacity/CapacityManagerImpl.java +++ b/server/src/com/cloud/capacity/CapacityManagerImpl.java @@ -47,6 +47,8 @@ import com.cloud.configuration.Config; import com.cloud.configuration.ConfigurationManager; import com.cloud.configuration.dao.ConfigurationDao; import com.cloud.dc.ClusterDetailsDao; +import com.cloud.dc.ClusterDetailsVO; +import com.cloud.dc.ClusterVO; import com.cloud.dc.dao.ClusterDao; import com.cloud.exception.ConnectionException; import com.cloud.host.Host; @@ -80,16 +82,21 @@ import com.cloud.utils.db.Transaction; import com.cloud.utils.fsm.StateListener; import com.cloud.vm.UserVmDetailVO; import com.cloud.vm.UserVmVO; +import com.cloud.vm.UserVmDetailVO; import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachine.Event; import com.cloud.vm.VirtualMachine.State; import com.cloud.vm.dao.UserVmDao; import com.cloud.vm.dao.UserVmDetailsDao; +import com.cloud.vm.dao.UserVmDetailsDao; import com.cloud.vm.dao.VMInstanceDao; import com.cloud.vm.snapshot.VMSnapshot; import com.cloud.vm.snapshot.VMSnapshotVO; import com.cloud.vm.snapshot.dao.VMSnapshotDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.log4j.Logger; +import org.springframework.stereotype.Component; @Component @Local(value = CapacityManager.class) @@ -125,11 +132,11 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager, protected UserVmDao _userVMDao; @Inject protected UserVmDetailsDao _userVmDetailsDao; + @Inject + ClusterDao _clusterDao; @Inject ClusterDetailsDao _clusterDetailsDao; - @Inject - ClusterDao _clusterDao; private int _vmCapacityReleaseInterval; private ScheduledExecutorService _executor; private boolean _stopped; @@ -530,10 +537,24 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager, s_logger.debug("Found " + vms.size() + " VMs on host " + host.getId()); } + ClusterVO cluster = _clusterDao.findById(host.getClusterId()); + ClusterDetailsVO clusterDetailCpu = _clusterDetailsDao.findDetail(cluster.getId(), "cpuOvercommitRatio"); + ClusterDetailsVO clusterDetailRam = _clusterDetailsDao.findDetail(cluster.getId(), "memoryOvercommitRatio"); + Float clusterCpuOvercommitRatio = Float.parseFloat(clusterDetailCpu.getValue()); + Float clusterRamOvercommitRatio = Float.parseFloat(clusterDetailRam.getValue()); + Float cpuOvercommitRatio = 1f; + Float ramOvercommitRatio = 1f; for (VMInstanceVO vm : vms) { + UserVmDetailVO vmDetailCpu = _userVmDetailsDao.findDetail(vm.getId(), "cpuOvercommitRatio"); + UserVmDetailVO vmDetailRam = _userVmDetailsDao.findDetail(vm.getId(),"memoryOvercommitRatio"); + if (vmDetailCpu != null ) { + //if vmDetail_cpu is not null it means it is running in a overcommited cluster. + cpuOvercommitRatio = Float.parseFloat(vmDetailCpu.getValue()); + ramOvercommitRatio = Float.parseFloat(vmDetailRam.getValue()); + } ServiceOffering so = offeringsMap.get(vm.getServiceOfferingId()); - usedMemory += so.getRamSize() * 1024L * 1024L; - usedCpu += so.getCpu() * so.getSpeed(); + usedMemory += ((so.getRamSize() * 1024L * 1024L)/ramOvercommitRatio)*clusterRamOvercommitRatio; + usedCpu += ((so.getCpu() * so.getSpeed())/cpuOvercommitRatio)*clusterCpuOvercommitRatio; } List vmsByLastHostId = _vmDao.listByLastHostId(host.getId()); @@ -543,9 +564,16 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager, for (VMInstanceVO vm : vmsByLastHostId) { long secondsSinceLastUpdate = (DateUtil.currentGMTTime().getTime() - vm.getUpdateTime().getTime()) / 1000; if (secondsSinceLastUpdate < _vmCapacityReleaseInterval) { + UserVmDetailVO vmDetailCpu = _userVmDetailsDao.findDetail(vm.getId(), "cpuOvercommitRatio"); + UserVmDetailVO vmDetailRam = _userVmDetailsDao.findDetail(vm.getId(),"memoryOvercommitRatio"); + if (vmDetailCpu != null ) { + //if vmDetail_cpu is not null it means it is running in a overcommited cluster. + cpuOvercommitRatio = Float.parseFloat(vmDetailCpu.getValue()); + ramOvercommitRatio = Float.parseFloat(vmDetailRam.getValue()); + } ServiceOffering so = offeringsMap.get(vm.getServiceOfferingId()); - reservedMemory += so.getRamSize() * 1024L * 1024L; - reservedCpu += so.getCpu() * so.getSpeed(); + reservedMemory += ((so.getRamSize() * 1024L * 1024L)/ramOvercommitRatio)*clusterRamOvercommitRatio; + reservedCpu += (so.getCpu() * so.getSpeed()/cpuOvercommitRatio)*clusterCpuOvercommitRatio; } else { // signal if not done already, that the VM has been stopped for skip.counting.hours, // hence capacity will not be reserved anymore. diff --git a/server/src/com/cloud/vm/VirtualMachineManagerImpl.java b/server/src/com/cloud/vm/VirtualMachineManagerImpl.java index 31bae47ed19..eedf4d20498 100755 --- a/server/src/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/server/src/com/cloud/vm/VirtualMachineManagerImpl.java @@ -180,6 +180,12 @@ import com.cloud.vm.snapshot.VMSnapshot; import com.cloud.vm.snapshot.VMSnapshotManager; import com.cloud.vm.snapshot.VMSnapshotVO; import com.cloud.vm.snapshot.dao.VMSnapshotDao; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; +import org.apache.cloudstack.engine.subsystem.api.storage.StoragePoolAllocator; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.log4j.Logger; import org.apache.cloudstack.affinity.dao.AffinityGroupVMMapDao; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; @@ -893,6 +899,21 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac Long cluster_id = dest.getCluster().getId(); ClusterDetailsVO cluster_detail_cpu = _clusterDetailsDao.findDetail(cluster_id,"cpuOvercommitRatio"); ClusterDetailsVO cluster_detail_ram = _clusterDetailsDao.findDetail(cluster_id,"memoryOvercommitRatio"); + //storing the value of overcommit in the vm_details table for doing a capacity check in case the cluster overcommit ratio is changed. + if (_uservmDetailsDao.findDetail(vm.getId(),"cpuOvercommitRatio") == null && ((Float.parseFloat(cluster_detail_cpu.getValue()) > 1f || Float.parseFloat(cluster_detail_ram.getValue()) > 1f)) ){ + UserVmDetailVO vmDetail_cpu = new UserVmDetailVO(vm.getId(), "cpuOvercommitRatio", cluster_detail_cpu.getValue()); + UserVmDetailVO vmDetail_ram = new UserVmDetailVO(vm.getId(), "memoryOvercommitRatio", cluster_detail_ram.getValue()); + _uservmDetailsDao.persist(vmDetail_cpu); + _uservmDetailsDao.persist(vmDetail_ram); + } + else if (((Float.parseFloat(cluster_detail_cpu.getValue()) > 1f || Float.parseFloat(cluster_detail_ram.getValue()) > 1f))) { + UserVmDetailVO vmDetail_cpu = _uservmDetailsDao.findDetail(vm.getId(), "cpuOvercommitRatio"); + vmDetail_cpu.setValue(cluster_detail_cpu.getValue()); + UserVmDetailVO vmDetail_ram = _uservmDetailsDao.findDetail(vm.getId(), "memoryOvercommitRatio"); + vmDetail_ram.setValue(cluster_detail_ram.getValue()); + _uservmDetailsDao.update(vmDetail_cpu.getId(), vmDetail_cpu); + _uservmDetailsDao.update(vmDetail_ram.getId(), vmDetail_ram); + } vmProfile.setCpuOvercommitRatio(Float.parseFloat(cluster_detail_cpu.getValue())); vmProfile.setMemoryOvercommitRatio(Float.parseFloat(cluster_detail_ram.getValue())); From 79af8b16b288229d815fe34c2efa8f78f9ae223c Mon Sep 17 00:00:00 2001 From: Wido den Hollander Date: Tue, 30 Jul 2013 15:06:29 +0200 Subject: [PATCH 007/763] CLOUDSTACK-3951: Don't require SSH access to KVM nodes when cancelling maintenance Conflicts: setup/db/db/schema-410to420.sql --- server/src/com/cloud/configuration/Config.java | 2 ++ server/src/com/cloud/resource/ResourceManagerImpl.java | 7 +++++++ setup/db/db/schema-410to420.sql | 3 +++ 3 files changed, 12 insertions(+) diff --git a/server/src/com/cloud/configuration/Config.java b/server/src/com/cloud/configuration/Config.java index ff484be0c1a..0d7269414e7 100755 --- a/server/src/com/cloud/configuration/Config.java +++ b/server/src/com/cloud/configuration/Config.java @@ -307,6 +307,8 @@ public enum Config { KvmPublicNetwork("Hidden", ManagementServer.class, String.class, "kvm.public.network.device", null, "Specify the public bridge on host for public network", null), KvmPrivateNetwork("Hidden", ManagementServer.class, String.class, "kvm.private.network.device", null, "Specify the private bridge on host for private network", null), KvmGuestNetwork("Hidden", ManagementServer.class, String.class, "kvm.guest.network.device", null, "Specify the private bridge on host for private network", null), + KvmSshToAgentEnabled("Advanced", ManagementServer.class, Boolean.class, "kvm.ssh.to.agent", "true", "Specify whether or not the management server is allowed to SSH into KVM Agents", null), + // Usage UsageExecutionTimezone("Usage", ManagementServer.class, String.class, "usage.execution.timezone", null, "The timezone to use for usage job execution time", null), UsageStatsJobAggregationRange("Usage", ManagementServer.class, Integer.class, "usage.stats.job.aggregation.range", "1440", "The range of time for aggregating the user statistics specified in minutes (e.g. 1440 for daily, 60 for hourly.", null), diff --git a/server/src/com/cloud/resource/ResourceManagerImpl.java b/server/src/com/cloud/resource/ResourceManagerImpl.java index 46fd808669d..878bfcda414 100755 --- a/server/src/com/cloud/resource/ResourceManagerImpl.java +++ b/server/src/com/cloud/resource/ResourceManagerImpl.java @@ -2114,6 +2114,13 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, // for kvm, need to log into kvm host, restart cloudstack-agent if (host.getHypervisorType() == HypervisorType.KVM) { + + boolean sshToAgent = Boolean.parseBoolean(_configDao.getValue(Config.KvmSshToAgentEnabled.key())); + if (!sshToAgent) { + s_logger.info("Configuration tells us not to SSH into Agents. Please restart the Agent (" + hostId + ") manually"); + return true; + } + _hostDao.loadDetails(host); String password = host.getDetail("password"); String username = host.getDetail("username"); diff --git a/setup/db/db/schema-410to420.sql b/setup/db/db/schema-410to420.sql index 9cdf83704af..1650e2fb174 100644 --- a/setup/db/db/schema-410to420.sql +++ b/setup/db/db/schema-410to420.sql @@ -2191,6 +2191,7 @@ ALTER TABLE `cloud`.`usage_event` ADD COLUMN `virtual_size` bigint unsigned; ALTER TABLE `cloud_usage`.`usage_event` ADD COLUMN `virtual_size` bigint unsigned; ALTER TABLE `cloud_usage`.`usage_storage` ADD COLUMN `virtual_size` bigint unsigned; ALTER TABLE `cloud_usage`.`cloud_usage` ADD COLUMN `virtual_size` bigint unsigned; + INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Network', 'DEFAULT', 'management-server', 'network.loadbalancer.haproxy.max.conn', '4096', 'Load Balancer(haproxy) maximum number of concurrent connections(global max)'); DROP TABLE IF EXISTS `cloud_usage`.`usage_vmsnapshot`; @@ -2208,3 +2209,5 @@ CREATE TABLE `cloud_usage`.`usage_vmsnapshot` ( ) ENGINE=InnoDB CHARSET=utf8; ALTER TABLE volumes ADD COLUMN vm_snapshot_chain_size bigint(20) unsigned; + +INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Advanced', 'DEFAULT', 'management-server', 'kvm.ssh.to.agent', 'true', 'Specify whether or not the management server is allowed to SSH into KVM Agents'); From 3cf09f337f0717b327013186e9775d8a858ed8df Mon Sep 17 00:00:00 2001 From: John Burwell Date: Wed, 31 Jul 2013 09:57:07 -0400 Subject: [PATCH 008/763] - CLOUDSTACK-3229: Passes the bucket name as "bucket" into the s3xen plugin --- .../com/cloud/hypervisor/xen/resource/CitrixResourceBase.java | 2 +- .../hypervisor/xen/resource/XenServerStorageProcessor.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java b/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java index 80bdf4299c0..1bf94bd2ebb 100644 --- a/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java +++ b/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java @@ -7500,7 +7500,7 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe serializeProperties(s3, S3Utils.ClientOptions.class)); parameters.addAll(Arrays.asList("operation", "put", "directory", dir, "filename", filename, "iSCSIFlag", - iSCSIFlag.toString(), "key", key)); + iSCSIFlag.toString(), "bucket", s3.getBucketName(), "key", key)); final String result = callHostPluginAsync(connection, "s3xen", "s3", wait, parameters.toArray(new String[parameters.size()])); diff --git a/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/XenServerStorageProcessor.java b/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/XenServerStorageProcessor.java index 69149ec89d9..c02c3b0bdc7 100644 --- a/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/XenServerStorageProcessor.java +++ b/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/XenServerStorageProcessor.java @@ -1129,7 +1129,7 @@ public class XenServerStorageProcessor implements StorageProcessor { serializeProperties(s3, S3Utils.ClientOptions.class)); parameters.addAll(Arrays.asList("operation", "put", "directory", dir, "filename", filename, "iSCSIFlag", - iSCSIFlag.toString(), "key", key)); + iSCSIFlag.toString(), "bucket", s3.getBucketName(), "key", key)); final String result = hypervisorResource.callHostPluginAsync(connection, "s3xen", "s3", wait, parameters.toArray(new String[parameters.size()])); From c68eb4d41e7129190ce70398a0dc8cbcdf070aa3 Mon Sep 17 00:00:00 2001 From: Chip Childers Date: Wed, 31 Jul 2013 11:08:43 -0400 Subject: [PATCH 009/763] Fixing setnextversion.sh --- tools/build/setnextversion.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/build/setnextversion.sh b/tools/build/setnextversion.sh index 71173a3a08c..7da3765704a 100755 --- a/tools/build/setnextversion.sh +++ b/tools/build/setnextversion.sh @@ -64,8 +64,8 @@ echo "found $currentversion" echo 'setting version numbers' mvn versions:set -DnewVersion=$version -P vmware -P developer -P systemvm -P simulator -P baremetal -P ucs -Dnonoss mv deps/XenServerJava/pom.xml.versionsBackup deps/XenServerJava/pom.xml -perl -pi -e 's/$ENV{'currentversion'}/$ENV{'version'}/' deps/XenServerJava/pom.xml -perl -pi -e 's/$ENV{'currentversion'}/$ENV{'version'}/' tools/apidoc/pom.xml +perl -pi -e "s/$currentversion/$version/" deps/XenServerJava/pom.xml +perl -pi -e "s/$currentversion/$version/" tools/apidoc/pom.xml git clean -f echo 'commit changes' From 0014381f38fe6887035c1d616c9f6d35da48dd96 Mon Sep 17 00:00:00 2001 From: Brian Federle Date: Wed, 31 Jul 2013 10:34:46 -0700 Subject: [PATCH 010/763] CLOUDSTACK-2944: Fix typo in addIpToNic dialog --- ui/scripts/network.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ui/scripts/network.js b/ui/scripts/network.js index d23dc626b1e..e5a52f75f44 100755 --- a/ui/scripts/network.js +++ b/ui/scripts/network.js @@ -1703,7 +1703,7 @@ title: 'label.acquire.new.secondary.ip', desc: 'message.acquire.ip.nic', fields: { - ipaddr: { + ipaddress: { label: 'label.ip.address' } } @@ -1718,8 +1718,8 @@ nicId: args.context.nics[0].id }; - if (args.data.ipaddr) { - dataObj.ipaddr = args.data.ipaddr; + if (args.data.ipaddress) { + dataObj.ipaddress = args.data.ipaddress; } $.ajax({ From f09df4c13682c1626fa9fa545df9e0f3c0a48320 Mon Sep 17 00:00:00 2001 From: Alena Prokharchyk Date: Wed, 31 Jul 2013 13:32:28 -0700 Subject: [PATCH 011/763] CLOUDSTACK-3974: 410-420 db upgrade - exclude is_inline field from the query when insert the entry for the F5 load balancer as this field was dropped as a part of 40-41 upgrade --- engine/schema/src/com/cloud/upgrade/dao/Upgrade410to420.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/engine/schema/src/com/cloud/upgrade/dao/Upgrade410to420.java b/engine/schema/src/com/cloud/upgrade/dao/Upgrade410to420.java index 8960c915c06..bfd8a60a97f 100644 --- a/engine/schema/src/com/cloud/upgrade/dao/Upgrade410to420.java +++ b/engine/schema/src/com/cloud/upgrade/dao/Upgrade410to420.java @@ -1413,7 +1413,7 @@ public class Upgrade410to420 implements DbUpgrade { try{ s_logger.debug("Adding F5 Big IP load balancer with host id " + hostId + " in to physical network" + physicalNetworkId); String insertF5 = "INSERT INTO `cloud`.`external_load_balancer_devices` (physical_network_id, host_id, provider_name, " + - "device_name, capacity, is_dedicated, device_state, allocation_state, is_inline, is_managed, uuid) VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; + "device_name, capacity, is_dedicated, device_state, allocation_state, is_managed, uuid) VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; pstmtUpdate = conn.prepareStatement(insertF5); pstmtUpdate.setLong(1, physicalNetworkId); pstmtUpdate.setLong(2, hostId); @@ -1424,8 +1424,7 @@ public class Upgrade410to420 implements DbUpgrade { pstmtUpdate.setString(7, "Enabled"); pstmtUpdate.setString(8, "Shared"); pstmtUpdate.setBoolean(9, false); - pstmtUpdate.setBoolean(10, false); - pstmtUpdate.setString(11, UUID.randomUUID().toString()); + pstmtUpdate.setString(10, UUID.randomUUID().toString()); pstmtUpdate.executeUpdate(); }catch (SQLException e) { throw new CloudRuntimeException("Exception while adding F5 load balancer device" , e); From fdc048442e98008da0ef5193180b69139f5e6313 Mon Sep 17 00:00:00 2001 From: Min Chen Date: Wed, 31 Jul 2013 14:00:20 -0700 Subject: [PATCH 012/763] CLOUDSTACK-3977: vm deployement is failing ;: Resource [Host:5] is unreachable: Host 5: Unable to send class org.apache.cloudstack.storage.command.CopyCommand because agent is in maintenance mode. --- .../cloudstack/storage/endpoint/DefaultEndPointSelector.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/storage/src/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java b/engine/storage/src/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java index c3f52ffcf29..fb041941442 100644 --- a/engine/storage/src/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java +++ b/engine/storage/src/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java @@ -55,7 +55,7 @@ public class DefaultEndPointSelector implements EndPointSelector { private static final Logger s_logger = Logger.getLogger(DefaultEndPointSelector.class); @Inject HostDao hostDao; - private String findOneHostOnPrimaryStorage = "select id from host where " + "status = 'Up' and type = 'Routing' "; + private String findOneHostOnPrimaryStorage = "select id from host where status = 'Up' and type = 'Routing' and resource_state = 'Enabled'"; protected boolean moveBetweenPrimaryImage(DataStore srcStore, DataStore destStore) { DataStoreRole srcRole = srcStore.getRole(); From 024dea93ae36327af830c9b2d7cb54598995caaf Mon Sep 17 00:00:00 2001 From: Sheng Yang Date: Wed, 31 Jul 2013 15:01:44 -0700 Subject: [PATCH 013/763] Automation: Fix test_vpc_routers.py Account cannot be cleaned per test case since it's setup in setUpClass() --- .../integration/component/test_vpc_routers.py | 70 ++++++++----------- 1 file changed, 31 insertions(+), 39 deletions(-) diff --git a/test/integration/component/test_vpc_routers.py b/test/integration/component/test_vpc_routers.py index be141fc6db8..9b772e41c67 100644 --- a/test/integration/component/test_vpc_routers.py +++ b/test/integration/component/test_vpc_routers.py @@ -599,6 +599,7 @@ class TestVPCRouterOneNetwork(cloudstackTestCase): TestVPCRouterOneNetwork, cls ).getClsTestClient().getApiClient() + cls._cleanup = [] cls.services = Services().services # Get Zone, Domain and templates cls.domain = get_domain(cls.apiclient, cls.services) @@ -615,11 +616,13 @@ class TestVPCRouterOneNetwork(cloudstackTestCase): cls.apiclient, cls.services["service_offering"] ) + cls._cleanup.append(cls.service_offering) cls.vpc_off = VpcOffering.create( cls.apiclient, cls.services["vpc_offering"] ) cls.vpc_off.update(cls.apiclient, state='Enabled') + cls._cleanup.append(cls.vpc_off) cls.account = Account.create( cls.apiclient, @@ -627,8 +630,7 @@ class TestVPCRouterOneNetwork(cloudstackTestCase): admin=True, domainid=cls.domain.id ) - cls._cleanup = [cls.account] - + cls._cleanup.insert(0, cls.account) cls.services["vpc"]["cidr"] = '10.1.1.1/16' cls.vpc = VPC.create( @@ -640,6 +642,31 @@ class TestVPCRouterOneNetwork(cloudstackTestCase): domainid=cls.account.domainid ) + private_gateway = PrivateGateway.create( + cls.apiclient, + gateway='10.1.3.1', + ipaddress='10.1.3.100', + netmask='255.255.255.0', + vlan=678, + vpcid=cls.vpc.id + ) + cls.gateways = PrivateGateway.list( + cls.apiclient, + id=private_gateway.id, + listall=True + ) + + static_route = StaticRoute.create( + cls.apiclient, + cidr='11.1.1.1/24', + gatewayid=private_gateway.id + ) + cls.static_routes = StaticRoute.list( + cls.apiclient, + id=static_route.id, + listall=True + ) + cls.nw_off = NetworkOffering.create( cls.apiclient, cls.services["network_offering"], @@ -695,6 +722,7 @@ class TestVPCRouterOneNetwork(cloudstackTestCase): domainid=cls.account.domainid, listall=True ) + public_ip_1 = PublicIPAddress.create( cls.apiclient, accountid=cls.account.name, @@ -757,7 +785,6 @@ class TestVPCRouterOneNetwork(cloudstackTestCase): vpcid=cls.vpc.id ) - lb_rule = LoadBalancerRule.create( cls.apiclient, cls.services["lbrule"], @@ -784,35 +811,6 @@ class TestVPCRouterOneNetwork(cloudstackTestCase): traffictype='Egress' ) - private_gateway = PrivateGateway.create( - cls.apiclient, - gateway='10.1.3.1', - ipaddress='10.1.3.100', - netmask='255.255.255.0', - vlan=678, - vpcid=cls.vpc.id - ) - cls.gateways = PrivateGateway.list( - cls.apiclient, - id=private_gateway.id, - listall=True - ) - static_route = StaticRoute.create( - cls.apiclient, - cidr='11.1.1.1/24', - gatewayid=private_gateway.id - ) - cls.static_routes = StaticRoute.list( - cls.apiclient, - id=static_route.id, - listall=True - ) - - cls._cleanup = [ - cls.service_offering, - cls.vpc_off - ] - @classmethod def tearDownClass(cls): try: @@ -824,13 +822,7 @@ class TestVPCRouterOneNetwork(cloudstackTestCase): def setUp(self): self.apiclient = self.testClient.getApiClient() - self.account = Account.create( - self.apiclient, - self.services["account"], - admin=True, - domainid=self.domain.id - ) - self.cleanup = [self.account] + self.cleanup = [] return def tearDown(self): From 4f111d284616419aa84f20b118cad7d81fbff580 Mon Sep 17 00:00:00 2001 From: Sheng Yang Date: Wed, 31 Jul 2013 15:10:01 -0700 Subject: [PATCH 014/763] Marvin: Add missing __init__ to StaticRoute --- tools/marvin/marvin/integration/lib/base.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/marvin/marvin/integration/lib/base.py b/tools/marvin/marvin/integration/lib/base.py index 48c49ab4b3f..bbf8440f725 100755 --- a/tools/marvin/marvin/integration/lib/base.py +++ b/tools/marvin/marvin/integration/lib/base.py @@ -3032,6 +3032,9 @@ class AffinityGroup: class StaticRoute: """Manage static route lifecycle""" + def __init__(self, items): + self.__dict__.update(items) + @classmethod def create(cls, apiclient, cidr, gatewayid): """Create static route""" From f9faf1aea2f3dbb47c60eaefcd999290dff837e7 Mon Sep 17 00:00:00 2001 From: Alena Prokharchyk Date: Wed, 31 Jul 2013 15:18:22 -0700 Subject: [PATCH 015/763] CLOUDSTACK-3982: listAccounts - count only UserVms when calculate total number of Running vms per account --- setup/db/db/schema-410to420.sql | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/setup/db/db/schema-410to420.sql b/setup/db/db/schema-410to420.sql index 1650e2fb174..3f25c3b7e59 100644 --- a/setup/db/db/schema-410to420.sql +++ b/setup/db/db/schema-410to420.sql @@ -2211,3 +2211,14 @@ CREATE TABLE `cloud_usage`.`usage_vmsnapshot` ( ALTER TABLE volumes ADD COLUMN vm_snapshot_chain_size bigint(20) unsigned; INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Advanced', 'DEFAULT', 'management-server', 'kvm.ssh.to.agent', 'true', 'Specify whether or not the management server is allowed to SSH into KVM Agents'); + +#update the account_vmstats_view - count only user vms +DROP VIEW IF EXISTS `cloud`.`account_vmstats_view`; +CREATE VIEW `cloud`.`account_vmstats_view` AS + SELECT + account_id, state, count(*) as vmcount + from + `cloud`.`vm_instance` + where + vm_type = 'User' + group by account_id , state; \ No newline at end of file From e3295a6af0d116d6de37dd40c9f5814bf100153b Mon Sep 17 00:00:00 2001 From: Edison Su Date: Wed, 31 Jul 2013 15:46:17 -0700 Subject: [PATCH 016/763] CLOUDSTACK-3950: the issue can be reproduced by: copy iso failed during the first time, for some reason, then copy the same iso again, the issue be triggered. --- .../storage/download/DownloadMonitorImpl.java | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/server/src/com/cloud/storage/download/DownloadMonitorImpl.java b/server/src/com/cloud/storage/download/DownloadMonitorImpl.java index f0550855bb5..0bb1a865358 100755 --- a/server/src/com/cloud/storage/download/DownloadMonitorImpl.java +++ b/server/src/com/cloud/storage/download/DownloadMonitorImpl.java @@ -103,10 +103,6 @@ public class DownloadMonitorImpl extends ManagerBase implements DownloadMonitor @Inject DataStoreManager storeMgr; - final Map _listenerTemplateMap = new ConcurrentHashMap(); - final Map _listenerVolMap = new ConcurrentHashMap(); - - @Override public boolean configure(String name, Map params) { final Map configs = _configDao.getConfiguration("ManagementServer", params); @@ -189,15 +185,6 @@ public class DownloadMonitorImpl extends ManagerBase implements DownloadMonitor dl.setCurrState(vmTemplateStore.getDownloadState()); } - DownloadListener old = null; - synchronized (_listenerTemplateMap) { - old = _listenerTemplateMap.put(vmTemplateStore, dl); - } - if (old != null) { - s_logger.info("abandon obsolete download listener"); - old.abandon(); - } - try { ep.sendMessageAsync(dcmd, new UploadListener.Callback(ep.getId(), dl)); } catch (Exception e) { @@ -270,13 +257,6 @@ public class DownloadMonitorImpl extends ManagerBase implements DownloadMonitor if (downloadJobExists) { dl.setCurrState(volumeHost.getDownloadState()); } - DownloadListener old = null; - synchronized (_listenerVolMap) { - old = _listenerVolMap.put(volumeHost, dl); - } - if (old != null) { - old.abandon(); - } try { ep.sendMessageAsync(dcmd, new UploadListener.Callback(ep.getId(), dl)); From b4770f1e012b1df5fd0e7e85ed6a7e93a93856b3 Mon Sep 17 00:00:00 2001 From: Murali Reddy Date: Thu, 1 Aug 2013 04:47:20 +0530 Subject: [PATCH 017/763] CLOUDSTACK-3731: [GSLB] deleteGlobalLoadBalancerRule fails with the java.lang.NumberFormatException While deleting LB monitor and GSLB service binding Nitro API fails with wierd NumberFormatException. Adding a workaround to delete the LB monitor after GSLB service is delted (which ensures intenrally LB monitor is delted). --- .../network/resource/NetscalerResource.java | 37 +++++++++++-------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/plugins/network-elements/netscaler/src/com/cloud/network/resource/NetscalerResource.java b/plugins/network-elements/netscaler/src/com/cloud/network/resource/NetscalerResource.java index eecae8a7ba5..c82fde2cde9 100644 --- a/plugins/network-elements/netscaler/src/com/cloud/network/resource/NetscalerResource.java +++ b/plugins/network-elements/netscaler/src/com/cloud/network/resource/NetscalerResource.java @@ -896,6 +896,7 @@ public class NetscalerResource implements ServerResource { // Add/Delete GSLB service corresponding the service running on each site String serviceName = GSLB.generateUniqueServiceName(siteName, servicePublicIp, servicePublicPort); + String monitorName = GSLB.generateGslbServiceMonitorName(servicePublicIp); if (!site.forRevoke()) { // create a 'gslbservice' object GSLB.createService(_netscalerService, serviceName, site.getServiceType(), @@ -908,24 +909,22 @@ public class NetscalerResource implements ServerResource { GSLB.createGslbServiceMonitor(_netscalerService, servicePublicIp, serviceName); // bind the monitor to the GSLB service - GSLB.createGslbServiceGslbMonitorBinding(_netscalerService, servicePublicIp, serviceName); + GSLB.createGslbServiceGslbMonitorBinding(_netscalerService, monitorName, serviceName); } else { - String monitorName = GSLB.generateGslbServiceMonitorName(servicePublicIp); - // delete GSLB service and GSLB monitor binding GSLB.deleteGslbServiceGslbMonitorBinding(_netscalerService, monitorName, serviceName); - // delete the GSLB service monitor - GSLB.deleteGslbServiceMonitor(_netscalerService, monitorName); - // Unbind GSLB service with GSLB virtual server GSLB.deleteVserverServiceBinding(_netscalerService, serviceName, vserverName); // delete 'gslbservice' object gslbservice service = GSLB.getServiceObject(_netscalerService, serviceName); GSLB.deleteService(_netscalerService, serviceName); + + // delete the GSLB service monitor + GSLB.deleteGslbServiceMonitor(_netscalerService, monitorName); } if (site.forRevoke()) { // delete the site if its for revoke @@ -954,10 +953,7 @@ public class NetscalerResource implements ServerResource { String monitorName = GSLB.generateGslbServiceMonitorName(servicePublicIp); // delete GSLB service and GSLB monitor binding - GSLB.deleteGslbServiceGslbMonitorBinding(_netscalerService, servicePublicIp, serviceName); - - // delete the GSLB service monitor - GSLB.deleteGslbServiceMonitor(_netscalerService, monitorName); + GSLB.deleteGslbServiceGslbMonitorBinding(_netscalerService, monitorName, serviceName); // remove binding between virtual server and services GSLB.deleteVserverServiceBinding(_netscalerService, serviceName, vserverName); @@ -967,6 +963,9 @@ public class NetscalerResource implements ServerResource { // delete GSLB site object GSLB.deleteSite(_netscalerService, siteName); + + // delete the GSLB service monitor + GSLB.deleteGslbServiceMonitor(_netscalerService, monitorName); } } @@ -1494,10 +1493,9 @@ public class NetscalerResource implements ServerResource { } } - private static void createGslbServiceGslbMonitorBinding(nitro_service nsService, String servicePublicIp, + private static void createGslbServiceGslbMonitorBinding(nitro_service nsService, String monitorName, String serviceName) { try { - String monitorName = GSLB.generateGslbServiceMonitorName(servicePublicIp); gslbservice_lbmonitor_binding monitorBinding = new gslbservice_lbmonitor_binding(); monitorBinding.set_monitor_name(monitorName); monitorBinding.set_servicename(serviceName); @@ -1513,10 +1511,19 @@ public class NetscalerResource implements ServerResource { String serviceName) { try { gslbservice_lbmonitor_binding[] monitorBindings = gslbservice_lbmonitor_binding.get(nsService, serviceName); - gslbservice_lbmonitor_binding.delete(nsService, monitorBindings); + if (monitorBindings != null && monitorBindings.length > 0) { + for (gslbservice_lbmonitor_binding binding : monitorBindings) { + if (binding.get_monitor_name().equalsIgnoreCase(monitorName)) { + s_logger.info("Found a binding between monitor " + binding.get_monitor_name() + " and " + + binding.get_servicename()); + gslbservice_lbmonitor_binding.delete(nsService, binding); + } + } + } } catch (Exception e) { - s_logger.warn("Failed to delet GSLB monitor " + monitorName + "and GSLB service " + serviceName + - " binding due to " + e.getMessage()); + s_logger.debug("Failed to delete GSLB monitor " + monitorName + " and GSLB service " + serviceName + + " binding due to " + e.getMessage() + " but moving on ..., will be cleaned up as part of GSLB " + + " service delete any way.."); } } From 2e25f3075e478a4ab55552f442e6c34a355580c6 Mon Sep 17 00:00:00 2001 From: Min Chen Date: Wed, 31 Jul 2013 16:17:42 -0700 Subject: [PATCH 018/763] CLOUDSTACK-3987: Setting extract.url.expiration.interval is not honored for S3 generated url, which is hard-coded to 1 hour. --- .../storage/datastore/driver/S3ImageStoreDriverImpl.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/plugins/storage/image/s3/src/org/apache/cloudstack/storage/datastore/driver/S3ImageStoreDriverImpl.java b/plugins/storage/image/s3/src/org/apache/cloudstack/storage/datastore/driver/S3ImageStoreDriverImpl.java index 934e5bd183e..ac681cc0206 100644 --- a/plugins/storage/image/s3/src/org/apache/cloudstack/storage/datastore/driver/S3ImageStoreDriverImpl.java +++ b/plugins/storage/image/s3/src/org/apache/cloudstack/storage/datastore/driver/S3ImageStoreDriverImpl.java @@ -36,6 +36,7 @@ import com.cloud.agent.api.to.S3TO; import com.cloud.configuration.Config; import com.cloud.configuration.dao.ConfigurationDao; import com.cloud.storage.Storage.ImageFormat; +import com.cloud.utils.NumbersUtil; import com.cloud.utils.S3Utils; public class S3ImageStoreDriverImpl extends BaseImageStoreDriverImpl { @@ -79,7 +80,11 @@ public class S3ImageStoreDriverImpl extends BaseImageStoreDriverImpl { s_logger.info("Generating pre-signed s3 entity extraction URL."); Date expiration = new Date(); long milliSeconds = expiration.getTime(); - milliSeconds += 1000 * 60 * 60; // expired after one hour. + + // get extract url expiration interval set in global configuration (in seconds) + String urlExpirationInterval = _configDao.getValue(Config.ExtractURLExpirationInterval.toString()); + int expirationInterval = NumbersUtil.parseInt(urlExpirationInterval, 14400); + milliSeconds += 1000 * expirationInterval; // expired after configured interval (in milliseconds) expiration.setTime(milliSeconds); URL s3url = S3Utils.generatePresignedUrl(s3, s3.getBucketName(), key, expiration); From dbc46d0677320288eba12dbe17a8c07767d7a5f4 Mon Sep 17 00:00:00 2001 From: Kelven Yang Date: Wed, 31 Jul 2013 16:25:36 -0700 Subject: [PATCH 019/763] Fix problems found in local test when JVM assertion is turned on --- engine/schema/src/com/cloud/dc/dao/VlanDaoImpl.java | 2 +- server/src/com/cloud/network/NetworkManagerImpl.java | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/engine/schema/src/com/cloud/dc/dao/VlanDaoImpl.java b/engine/schema/src/com/cloud/dc/dao/VlanDaoImpl.java index eb3bde9d005..6f5a01f1476 100755 --- a/engine/schema/src/com/cloud/dc/dao/VlanDaoImpl.java +++ b/engine/schema/src/com/cloud/dc/dao/VlanDaoImpl.java @@ -347,7 +347,7 @@ public class VlanDaoImpl extends GenericDaoBase implements VlanDao @Override public List listZoneWideNonDedicatedVlans(long zoneId) { SearchCriteria sc = ZoneWideNonDedicatedVlanSearch.create(); - sc.setParameters("ZoneWideNonDedicatedVlanSearch", "zoneId", zoneId); + sc.setParameters("zoneId", zoneId); return listBy(sc); } diff --git a/server/src/com/cloud/network/NetworkManagerImpl.java b/server/src/com/cloud/network/NetworkManagerImpl.java index c9380f880f4..d2761b0184f 100755 --- a/server/src/com/cloud/network/NetworkManagerImpl.java +++ b/server/src/com/cloud/network/NetworkManagerImpl.java @@ -1488,6 +1488,8 @@ public class NetworkManagerImpl extends ManagerBase implements NetworkManager, L AssignIpAddressFromPodVlanSearch = _ipAddressDao.createSearchBuilder(); AssignIpAddressFromPodVlanSearch.and("dc", AssignIpAddressFromPodVlanSearch.entity().getDataCenterId(), Op.EQ); AssignIpAddressFromPodVlanSearch.and("allocated", AssignIpAddressFromPodVlanSearch.entity().getAllocatedTime(), Op.NULL); + AssignIpAddressFromPodVlanSearch.and("vlanId", AssignIpAddressFromPodVlanSearch.entity().getVlanId(), Op.IN); + SearchBuilder podVlanSearch = _vlanDao.createSearchBuilder(); podVlanSearch.and("type", podVlanSearch.entity().getVlanType(), Op.EQ); podVlanSearch.and("networkId", podVlanSearch.entity().getNetworkId(), Op.EQ); @@ -1495,6 +1497,8 @@ public class NetworkManagerImpl extends ManagerBase implements NetworkManager, L podVlanMapSB.and("podId", podVlanMapSB.entity().getPodId(), Op.EQ); AssignIpAddressFromPodVlanSearch.join("podVlanMapSB", podVlanMapSB, podVlanMapSB.entity().getVlanDbId(), AssignIpAddressFromPodVlanSearch.entity().getVlanId(), JoinType.INNER); AssignIpAddressFromPodVlanSearch.join("vlan", podVlanSearch, podVlanSearch.entity().getId(), AssignIpAddressFromPodVlanSearch.entity().getVlanId(), JoinType.INNER); + + AssignIpAddressFromPodVlanSearch.done(); _executor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("Network-Scavenger")); From da124c2d8053442dac6b73c633b1b78735570766 Mon Sep 17 00:00:00 2001 From: Edison Su Date: Wed, 31 Jul 2013 17:01:04 -0700 Subject: [PATCH 020/763] CLOUDSTACK-3976: delete parent snapshot if child snapshots are deleted --- .../storage/snapshot/SnapshotObject.java | 1 + .../snapshot/XenserverSnapshotStrategy.java | 15 ++++++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotObject.java b/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotObject.java index 2fc576b8b8f..c8a0bc8a794 100644 --- a/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotObject.java +++ b/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotObject.java @@ -107,6 +107,7 @@ public class SnapshotObject implements SnapshotInfo { .create(SnapshotDataStoreVO.class); sc.addAnd(sc.getEntity().getDataStoreId(), Op.EQ, this.store.getId()); sc.addAnd(sc.getEntity().getRole(), Op.EQ, this.store.getRole()); + sc.addAnd(sc.getEntity().getState(), Op.NIN, State.Destroying, State.Destroyed, State.Error); sc.addAnd(sc.getEntity().getParentSnapshotId(), Op.EQ, this.getId()); SnapshotDataStoreVO vo = sc.find(); if (vo == null) { diff --git a/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/XenserverSnapshotStrategy.java b/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/XenserverSnapshotStrategy.java index ebbce5b16a4..c470bfe0593 100644 --- a/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/XenserverSnapshotStrategy.java +++ b/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/XenserverSnapshotStrategy.java @@ -140,7 +140,20 @@ public class XenserverSnapshotStrategy extends SnapshotStrategyBase { } s_logger.debug("Snapshot: " + snapshot.getId() + " doesn't have children, so it's ok to delete it and its parents"); SnapshotInfo parent = snapshot.getParent(); - result = this.snapshotSvr.deleteSnapshot(snapshot); + boolean deleted = false; + if (parent != null) { + if (parent.getPath() != null && parent.getPath().equalsIgnoreCase(snapshot.getPath())) { + //NOTE: if both snapshots share the same path, it's for xenserver's empty delta snapshot. We can't delete the snapshot on the backend, as parent snapshot still reference to it + //Instead, mark it as destroyed in the db. + s_logger.debug("for empty delta snapshot, only mark it as destroyed in db"); + snapshot.processEvent(Event.DestroyRequested); + snapshot.processEvent(Event.OperationSuccessed); + deleted = true; + } + } + if (!deleted) { + result = this.snapshotSvr.deleteSnapshot(snapshot); + } snapshot = parent; } return result; From a747ee508bcc6786432ed68e900f3d1fd35d05e0 Mon Sep 17 00:00:00 2001 From: Edison Su Date: Wed, 31 Jul 2013 17:42:35 -0700 Subject: [PATCH 021/763] CLOUDSTACK-3976: if the snapshot is empty and it's the last one in the chain, we need to take full snapshot for the snapshots in sequence --- .../snapshot/XenserverSnapshotStrategy.java | 57 ++++++++++++------- 1 file changed, 35 insertions(+), 22 deletions(-) diff --git a/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/XenserverSnapshotStrategy.java b/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/XenserverSnapshotStrategy.java index c470bfe0593..3e6c508a2ea 100644 --- a/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/XenserverSnapshotStrategy.java +++ b/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/XenserverSnapshotStrategy.java @@ -130,31 +130,44 @@ public class XenserverSnapshotStrategy extends SnapshotStrategyBase { protected boolean deleteSnapshotChain(SnapshotInfo snapshot) { s_logger.debug("delete snapshot chain for snapshot: " + snapshot.getId()); boolean result = false; - while (snapshot != null && (snapshot.getState() == Snapshot.State.Destroying || snapshot.getState() - == Snapshot.State.Destroyed || snapshot.getState() == Snapshot.State.Error)) { - SnapshotInfo child = snapshot.getChild(); + boolean resultIsSet = false; //need to track, the snapshot itself is deleted or not. + try { + while (snapshot != null && (snapshot.getState() == Snapshot.State.Destroying || snapshot.getState() + == Snapshot.State.Destroyed || snapshot.getState() == Snapshot.State.Error)) { + SnapshotInfo child = snapshot.getChild(); - if (child != null) { - s_logger.debug("the snapshot has child, can't delete it on the storage"); - break; - } - s_logger.debug("Snapshot: " + snapshot.getId() + " doesn't have children, so it's ok to delete it and its parents"); - SnapshotInfo parent = snapshot.getParent(); - boolean deleted = false; - if (parent != null) { - if (parent.getPath() != null && parent.getPath().equalsIgnoreCase(snapshot.getPath())) { - //NOTE: if both snapshots share the same path, it's for xenserver's empty delta snapshot. We can't delete the snapshot on the backend, as parent snapshot still reference to it - //Instead, mark it as destroyed in the db. - s_logger.debug("for empty delta snapshot, only mark it as destroyed in db"); - snapshot.processEvent(Event.DestroyRequested); - snapshot.processEvent(Event.OperationSuccessed); - deleted = true; + if (child != null) { + s_logger.debug("the snapshot has child, can't delete it on the storage"); + break; } + s_logger.debug("Snapshot: " + snapshot.getId() + " doesn't have children, so it's ok to delete it and its parents"); + SnapshotInfo parent = snapshot.getParent(); + boolean deleted = false; + if (parent != null) { + if (parent.getPath() != null && parent.getPath().equalsIgnoreCase(snapshot.getPath())) { + //NOTE: if both snapshots share the same path, it's for xenserver's empty delta snapshot. We can't delete the snapshot on the backend, as parent snapshot still reference to it + //Instead, mark it as destroyed in the db. + s_logger.debug("for empty delta snapshot, only mark it as destroyed in db"); + snapshot.processEvent(Event.DestroyRequested); + snapshot.processEvent(Event.OperationSuccessed); + deleted = true; + if (!resultIsSet) { + result = true; + resultIsSet = true; + } + } + } + if (!deleted) { + boolean r = this.snapshotSvr.deleteSnapshot(snapshot); + if (!resultIsSet) { + result = r; + resultIsSet = true; + } + } + snapshot = parent; } - if (!deleted) { - result = this.snapshotSvr.deleteSnapshot(snapshot); - } - snapshot = parent; + } catch (Exception e) { + s_logger.debug("delete snapshot failed: ", e); } return result; } From 7acb023aad5a2343b6ed0953c19f4c49bfc293df Mon Sep 17 00:00:00 2001 From: Kelven Yang Date: Wed, 31 Jul 2013 17:58:15 -0700 Subject: [PATCH 022/763] More fixes found in test cases when JVM assertion check is turned on --- engine/schema/src/com/cloud/host/dao/HostDaoImpl.java | 2 +- engine/schema/src/com/cloud/vm/dao/NicDaoImpl.java | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/engine/schema/src/com/cloud/host/dao/HostDaoImpl.java b/engine/schema/src/com/cloud/host/dao/HostDaoImpl.java index dd26941aa00..e22274e2967 100755 --- a/engine/schema/src/com/cloud/host/dao/HostDaoImpl.java +++ b/engine/schema/src/com/cloud/host/dao/HostDaoImpl.java @@ -392,7 +392,7 @@ public class HostDaoImpl extends GenericDaoBase implements HostDao @Override public List listByDataCenterId(long id) { SearchCriteria sc = DcSearch.create(); - sc.setParameters("dcId", id); + sc.setParameters("dc", id); sc.setParameters("status", Status.Up); sc.setParameters("type", Host.Type.Routing); sc.setParameters("resourceState", ResourceState.Enabled); diff --git a/engine/schema/src/com/cloud/vm/dao/NicDaoImpl.java b/engine/schema/src/com/cloud/vm/dao/NicDaoImpl.java index d6433a45ffb..2868b4f6ab9 100644 --- a/engine/schema/src/com/cloud/vm/dao/NicDaoImpl.java +++ b/engine/schema/src/com/cloud/vm/dao/NicDaoImpl.java @@ -54,6 +54,7 @@ public class NicDaoImpl extends GenericDaoBase implements NicDao { AllFieldsSearch.and("broadcastUri", AllFieldsSearch.entity().getBroadcastUri(), Op.EQ); AllFieldsSearch.and("secondaryip", AllFieldsSearch.entity().getSecondaryIp(), Op.EQ); AllFieldsSearch.and("nicid", AllFieldsSearch.entity().getId(), Op.EQ); + AllFieldsSearch.and("strategy", AllFieldsSearch.entity().getReservationStrategy(), Op.EQ); AllFieldsSearch.done(); IpSearch = createSearchBuilder(String.class); From fda055388e3776cb630e167340f887b4f329339f Mon Sep 17 00:00:00 2001 From: Sheng Yang Date: Wed, 31 Jul 2013 18:27:11 -0700 Subject: [PATCH 023/763] Automation: Fix SSH connection test And increase default retry to 10 times, each 30 seconds. --- tools/marvin/marvin/integration/lib/utils.py | 2 +- tools/marvin/marvin/remoteSSHClient.py | 24 +++++++++++--------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/tools/marvin/marvin/integration/lib/utils.py b/tools/marvin/marvin/integration/lib/utils.py index 2da9272c7aa..6d101327121 100644 --- a/tools/marvin/marvin/integration/lib/utils.py +++ b/tools/marvin/marvin/integration/lib/utils.py @@ -109,7 +109,7 @@ def cleanup_resources(api_client, resources): obj.delete(api_client) -def is_server_ssh_ready(ipaddress, port, username, password, retries=5, timeout=30, keyPairFileLocation=None): +def is_server_ssh_ready(ipaddress, port, username, password, retries=10, timeout=30, keyPairFileLocation=None): """Return ssh handle else wait till sshd is running""" try: ssh = remoteSSHClient( diff --git a/tools/marvin/marvin/remoteSSHClient.py b/tools/marvin/marvin/remoteSSHClient.py index e0ead9315a9..fea9b125d19 100644 --- a/tools/marvin/marvin/remoteSSHClient.py +++ b/tools/marvin/marvin/remoteSSHClient.py @@ -17,7 +17,6 @@ import paramiko import time -import socket import cloudstackException import contextlib import logging @@ -25,7 +24,7 @@ from contextlib import closing class remoteSSHClient(object): - def __init__(self, host, port, user, passwd, retries=5, delay=30, + def __init__(self, host, port, user, passwd, retries=10, delay=30, log_lvl=logging.INFO, keyPairFileLocation=None): self.host = host self.port = port @@ -40,7 +39,7 @@ class remoteSSHClient(object): self.logger.addHandler(ch) retry_count = retries - while True: + while retry_count >= 0: try: if keyPairFileLocation is None: self.ssh.connect(str(host), int(port), user, passwd) @@ -58,17 +57,20 @@ class remoteSSHClient(object): (str(host), user, keyPairFileLocation)) self.logger.debug("SSH connect: %s@%s with passwd %s" % (user, str(host), passwd)) - except (paramiko.SSHException, paramiko.ChannelException, socket.error) as se: + #except paramiko.AuthenticationException, authEx: + # raise cloudstackException. \ + # InvalidParameterException("Invalid credentials to " + # + "login to %s on port %s" % + # (str(host), port)) + except Exception as se: if retry_count == 0: raise cloudstackException. \ InvalidParameterException(repr(se)) - retry_count = retry_count - 1 - time.sleep(delay) - except paramiko.AuthenticationException, authEx: - raise cloudstackException. \ - InvalidParameterException("Invalid credentials to " - + "login to %s on port %s" % - (str(host), port)) + else: + return + + retry_count = retry_count - 1 + time.sleep(delay) def execute(self, command): stdin, stdout, stderr = self.ssh.exec_command(command) From af3704cd32077a22b55adbf4b59ebb9fb4b8242b Mon Sep 17 00:00:00 2001 From: Sheng Yang Date: Wed, 31 Jul 2013 19:08:17 -0700 Subject: [PATCH 024/763] Automation: Fix test_vpc_network_pfrules.py Need to wait VM to boot up before stop the VR. Also test 4 is identical to test 3, fix it. --- .../component/test_vpc_network_pfrules.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/test/integration/component/test_vpc_network_pfrules.py b/test/integration/component/test_vpc_network_pfrules.py index 9e6243b6ed8..274012da6a8 100644 --- a/test/integration/component/test_vpc_network_pfrules.py +++ b/test/integration/component/test_vpc_network_pfrules.py @@ -557,6 +557,10 @@ class TestVPCNetworkPFRules(cloudstackTestCase): network_2 = self.create_network(self.services["network_offering_no_lb"], '10.1.2.1') vm_1 = self.deployvm_in_network(network_1) vm_2 = self.deployvm_in_network(network_2) + + # wait until VM is up before stop the VR + time.sleep(120) + public_ip_1 = self.acquire_publicip(network_1) public_ip_2 = self.acquire_publicip(network_2) router = self.stop_vpcrouter() @@ -581,8 +585,7 @@ class TestVPCNetworkPFRules(cloudstackTestCase): # 6. Deploy vm2 in network2. # 7. Use the Create PF rule for vm1 in network1. # 8. Use the Create PF rule for vm2 in network2. - # 9. Start VPC Virtual Router. - # 10. Successfully ssh into the Guest VM1 and VM2 using the PF rule + # 9. Successfully ssh into the Guest VM1 and VM2 using the PF rule network_1 = self.create_network(self.services["network_offering"]) network_2 = self.create_network(self.services["network_offering_no_lb"], '10.1.2.1') @@ -590,10 +593,8 @@ class TestVPCNetworkPFRules(cloudstackTestCase): vm_2 = self.deployvm_in_network(network_2) public_ip_1 = self.acquire_publicip(network_1) public_ip_2 = self.acquire_publicip(network_2) - router = self.stop_vpcrouter() self.create_natrule(vm_1, public_ip_1, network_1) self.create_natrule(vm_2, public_ip_2, network_2) - self.start_vpcrouter(router) self.check_ssh_into_vm(vm_1, public_ip_1, testnegative=False) self.check_ssh_into_vm(vm_2, public_ip_2, testnegative=False) return @@ -621,7 +622,6 @@ class TestVPCNetworkPFRules(cloudstackTestCase): public_ip_1 = self.acquire_publicip(network_1) self.create_natrule(vm_1, public_ip_1, network_1) http_rule = self.create_natrule(vm_1, public_ip_1, network_1, self.services["http_rule"]) - #http_rule = self.create_egress_Internet_Rule(network_1) self.check_ssh_into_vm(vm_1, public_ip_1, testnegative=False) self.check_wget_from_vm(vm_1, public_ip_1, testnegative=False) router = self.stop_vpcrouter() @@ -651,7 +651,6 @@ class TestVPCNetworkPFRules(cloudstackTestCase): public_ip_1 = self.acquire_publicip(network_1) self.create_natrule(vm_1, public_ip_1, network_1) http_rule=self.create_natrule(vm_1, public_ip_1, network_1, self.services["http_rule"]) - #http_rule = self.create_egress_Internet_Rule(network_1) self.check_ssh_into_vm(vm_1, public_ip_1, testnegative=False) self.check_wget_from_vm(vm_1, public_ip_1, testnegative=False) http_rule.delete(self.apiclient) @@ -682,7 +681,6 @@ class TestVPCNetworkPFRules(cloudstackTestCase): public_ip_1 = self.acquire_publicip(network_1) nat_rule = self.create_natrule(vm_1, public_ip_1, network_1) http_rule = self.create_natrule(vm_1, public_ip_1, network_1, self.services["http_rule"]) - #http_rule = self.create_egress_Internet_Rule(network_1) self.check_ssh_into_vm(vm_1, public_ip_1, testnegative=False) self.check_wget_from_vm(vm_1, public_ip_1, testnegative=False) router = self.stop_vpcrouter() @@ -715,7 +713,6 @@ class TestVPCNetworkPFRules(cloudstackTestCase): public_ip_1 = self.acquire_publicip(network_1) nat_rule = self.create_natrule(vm_1, public_ip_1, network_1) http_rule = self.create_natrule(vm_1, public_ip_1, network_1, self.services["http_rule"]) - #http_rule = self.create_egress_Internet_Rule(network_1) self.check_ssh_into_vm(vm_1, public_ip_1, testnegative=False) self.check_wget_from_vm(vm_1, public_ip_1, testnegative=False) http_rule.delete(self.apiclient) From 59492fae109a313b75d1e286c9b30199b150fdd3 Mon Sep 17 00:00:00 2001 From: John Burwell Date: Wed, 31 Jul 2013 23:14:28 -0400 Subject: [PATCH 025/763] - CLOUDSTACK-3229: Properly serialize the https property to the s3Xen plugin - Extracts the duplicated serializeProperties methods to ReflectUtils#flattenProperties - Adds unit tests for ReflectUtils#flattenProperties --- .../xen/resource/CitrixResourceBase.java | 214 +++++++----------- .../resource/XenServerStorageProcessor.java | 60 +---- scripts/vm/hypervisor/xenserver/s3xen | 2 +- utils/src/com/cloud/utils/ReflectUtil.java | 88 ++++++- .../test/com/cloud/utils/ReflectUtilTest.java | 91 ++++++++ 5 files changed, 261 insertions(+), 194 deletions(-) create mode 100644 utils/test/com/cloud/utils/ReflectUtilTest.java diff --git a/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java b/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java index 1bf94bd2ebb..af408d8fb69 100644 --- a/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java +++ b/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java @@ -16,92 +16,6 @@ // under the License. package com.cloud.hypervisor.xen.resource; -import com.google.gson.Gson; - -import java.beans.BeanInfo; -import java.beans.IntrospectionException; -import java.beans.Introspector; -import java.beans.PropertyDescriptor; -import java.io.BufferedReader; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.StringReader; -import java.lang.reflect.InvocationTargetException; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.URL; -import java.net.URLConnection; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.Queue; -import java.util.Random; -import java.util.Set; -import java.util.UUID; - -import javax.ejb.Local; -import javax.naming.ConfigurationException; -import javax.xml.parsers.DocumentBuilderFactory; - -import com.cloud.agent.api.to.DhcpTO; -import org.apache.commons.codec.binary.Base64; -import org.apache.log4j.Logger; -import org.apache.xmlrpc.XmlRpcException; -import org.w3c.dom.Document; -import org.w3c.dom.Node; -import org.w3c.dom.NodeList; -import org.xml.sax.InputSource; - -import com.trilead.ssh2.SCPClient; -import com.xensource.xenapi.Bond; -import com.xensource.xenapi.Connection; -import com.xensource.xenapi.Console; -import com.xensource.xenapi.Host; -import com.xensource.xenapi.HostCpu; -import com.xensource.xenapi.HostMetrics; -import com.xensource.xenapi.Network; -import com.xensource.xenapi.PBD; -import com.xensource.xenapi.PIF; -import com.xensource.xenapi.PIF.Record; -import com.xensource.xenapi.Pool; -import com.xensource.xenapi.SR; -import com.xensource.xenapi.Session; -import com.xensource.xenapi.Task; -import com.xensource.xenapi.Types; -import com.xensource.xenapi.Types.BadAsyncResult; -import com.xensource.xenapi.Types.BadServerResponse; -import com.xensource.xenapi.Types.ConsoleProtocol; -import com.xensource.xenapi.Types.IpConfigurationMode; -import com.xensource.xenapi.Types.OperationNotAllowed; -import com.xensource.xenapi.Types.SrFull; -import com.xensource.xenapi.Types.VbdType; -import com.xensource.xenapi.Types.VmBadPowerState; -import com.xensource.xenapi.Types.VmPowerState; -import com.xensource.xenapi.Types.XenAPIException; -import com.xensource.xenapi.VBD; -import com.xensource.xenapi.VBDMetrics; -import com.xensource.xenapi.VDI; -import com.xensource.xenapi.VIF; -import com.xensource.xenapi.VLAN; -import com.xensource.xenapi.VM; -import com.xensource.xenapi.VMGuestMetrics; -import com.xensource.xenapi.XenAPIObject; - -import org.apache.cloudstack.storage.command.StorageSubSystemCommand; -import org.apache.cloudstack.storage.to.TemplateObjectTO; -import org.apache.cloudstack.storage.to.VolumeObjectTO; - import com.cloud.agent.IAgentControl; import com.cloud.agent.api.Answer; import com.cloud.agent.api.AttachIsoCommand; @@ -242,6 +156,7 @@ import com.cloud.agent.api.storage.ResizeVolumeAnswer; import com.cloud.agent.api.storage.ResizeVolumeCommand; import com.cloud.agent.api.to.DataStoreTO; import com.cloud.agent.api.to.DataTO; +import com.cloud.agent.api.to.DhcpTO; import com.cloud.agent.api.to.DiskTO; import com.cloud.agent.api.to.FirewallRuleTO; import com.cloud.agent.api.to.IpAddressTO; @@ -299,6 +214,82 @@ import com.cloud.vm.DiskProfile; import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachine.State; import com.cloud.vm.snapshot.VMSnapshot; +import com.google.gson.Gson; +import com.trilead.ssh2.SCPClient; +import com.xensource.xenapi.Bond; +import com.xensource.xenapi.Connection; +import com.xensource.xenapi.Console; +import com.xensource.xenapi.Host; +import com.xensource.xenapi.HostCpu; +import com.xensource.xenapi.HostMetrics; +import com.xensource.xenapi.Network; +import com.xensource.xenapi.PBD; +import com.xensource.xenapi.PIF; +import com.xensource.xenapi.PIF.Record; +import com.xensource.xenapi.Pool; +import com.xensource.xenapi.SR; +import com.xensource.xenapi.Session; +import com.xensource.xenapi.Task; +import com.xensource.xenapi.Types; +import com.xensource.xenapi.Types.BadAsyncResult; +import com.xensource.xenapi.Types.BadServerResponse; +import com.xensource.xenapi.Types.ConsoleProtocol; +import com.xensource.xenapi.Types.IpConfigurationMode; +import com.xensource.xenapi.Types.OperationNotAllowed; +import com.xensource.xenapi.Types.SrFull; +import com.xensource.xenapi.Types.VbdType; +import com.xensource.xenapi.Types.VmBadPowerState; +import com.xensource.xenapi.Types.VmPowerState; +import com.xensource.xenapi.Types.XenAPIException; +import com.xensource.xenapi.VBD; +import com.xensource.xenapi.VBDMetrics; +import com.xensource.xenapi.VDI; +import com.xensource.xenapi.VIF; +import com.xensource.xenapi.VLAN; +import com.xensource.xenapi.VM; +import com.xensource.xenapi.VMGuestMetrics; +import com.xensource.xenapi.XenAPIObject; +import org.apache.cloudstack.storage.command.StorageSubSystemCommand; +import org.apache.cloudstack.storage.to.TemplateObjectTO; +import org.apache.cloudstack.storage.to.VolumeObjectTO; +import org.apache.commons.codec.binary.Base64; +import org.apache.log4j.Logger; +import org.apache.xmlrpc.XmlRpcException; +import org.w3c.dom.Document; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; +import org.xml.sax.InputSource; + +import javax.ejb.Local; +import javax.naming.ConfigurationException; +import javax.xml.parsers.DocumentBuilderFactory; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.StringReader; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.net.URLConnection; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Queue; +import java.util.Random; +import java.util.Set; +import java.util.UUID; + +import static com.cloud.utils.ReflectUtil.flattenProperties; /** * CitrixResourceBase encapsulates the calls to the XenServer Xapi process @@ -7437,53 +7428,6 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe return new BackupSnapshotAnswer(cmd, success, details, snapshotBackupUuid, fullbackup); } - private static List serializeProperties(final Object object, - final Class propertySet) { - - assert object != null; - assert propertySet != null; - assert propertySet.isAssignableFrom(object.getClass()); - - try { - - final BeanInfo beanInfo = Introspector.getBeanInfo(propertySet); - final PropertyDescriptor[] descriptors = beanInfo - .getPropertyDescriptors(); - - final List serializedProperties = new ArrayList(); - for (final PropertyDescriptor descriptor : descriptors) { - - serializedProperties.add(descriptor.getName()); - final Object value = descriptor.getReadMethod().invoke(object); - serializedProperties.add(value != null ? value.toString() - : "null"); - - } - - return Collections.unmodifiableList(serializedProperties); - - } catch (IntrospectionException e) { - s_logger.warn( - "Ignored IntrospectionException when serializing class " - + object.getClass().getCanonicalName(), e); - } catch (IllegalArgumentException e) { - s_logger.warn( - "Ignored IllegalArgumentException when serializing class " - + object.getClass().getCanonicalName(), e); - } catch (IllegalAccessException e) { - s_logger.warn( - "Ignored IllegalAccessException when serializing class " - + object.getClass().getCanonicalName(), e); - } catch (InvocationTargetException e) { - s_logger.warn( - "Ignored InvocationTargetException when serializing class " - + object.getClass().getCanonicalName(), e); - } - - return Collections.emptyList(); - - } - private boolean backupSnapshotToS3(final Connection connection, final S3TO s3, final String srUuid, final String snapshotUuid, final Boolean iSCSIFlag, final int wait) { @@ -7496,8 +7440,8 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe try { - final List parameters = new ArrayList( - serializeProperties(s3, S3Utils.ClientOptions.class)); + final List parameters = flattenProperties(s3, + S3Utils.ClientOptions.class); parameters.addAll(Arrays.asList("operation", "put", "directory", dir, "filename", filename, "iSCSIFlag", iSCSIFlag.toString(), "bucket", s3.getBucketName(), "key", key)); diff --git a/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/XenServerStorageProcessor.java b/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/XenServerStorageProcessor.java index c02c3b0bdc7..68a07c07ff1 100644 --- a/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/XenServerStorageProcessor.java +++ b/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/XenServerStorageProcessor.java @@ -71,22 +71,17 @@ import org.apache.cloudstack.storage.to.VolumeObjectTO; import org.apache.log4j.Logger; import org.apache.xmlrpc.XmlRpcException; -import java.beans.BeanInfo; -import java.beans.IntrospectionException; -import java.beans.Introspector; -import java.beans.PropertyDescriptor; import java.io.File; -import java.lang.reflect.InvocationTargetException; import java.net.URI; -import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; import java.util.UUID; +import static com.cloud.utils.ReflectUtil.flattenProperties; + public class XenServerStorageProcessor implements StorageProcessor { private static final Logger s_logger = Logger.getLogger(XenServerStorageProcessor.class); protected CitrixResourceBase hypervisorResource; @@ -1066,53 +1061,6 @@ public class XenServerStorageProcessor implements StorageProcessor { return lfilename; } - private static List serializeProperties(final Object object, - final Class propertySet) { - - assert object != null; - assert propertySet != null; - assert propertySet.isAssignableFrom(object.getClass()); - - try { - - final BeanInfo beanInfo = Introspector.getBeanInfo(propertySet); - final PropertyDescriptor[] descriptors = beanInfo - .getPropertyDescriptors(); - - final List serializedProperties = new ArrayList(); - for (final PropertyDescriptor descriptor : descriptors) { - - serializedProperties.add(descriptor.getName()); - final Object value = descriptor.getReadMethod().invoke(object); - serializedProperties.add(value != null ? value.toString() - : "null"); - - } - - return Collections.unmodifiableList(serializedProperties); - - } catch (IntrospectionException e) { - s_logger.warn( - "Ignored IntrospectionException when serializing class " - + object.getClass().getCanonicalName(), e); - } catch (IllegalArgumentException e) { - s_logger.warn( - "Ignored IllegalArgumentException when serializing class " - + object.getClass().getCanonicalName(), e); - } catch (IllegalAccessException e) { - s_logger.warn( - "Ignored IllegalAccessException when serializing class " - + object.getClass().getCanonicalName(), e); - } catch (InvocationTargetException e) { - s_logger.warn( - "Ignored InvocationTargetException when serializing class " - + object.getClass().getCanonicalName(), e); - } - - return Collections.emptyList(); - - } - private boolean backupSnapshotToS3(final Connection connection, final S3TO s3, final String srUuid, final String snapshotUuid, final Boolean iSCSIFlag, final int wait) { @@ -1125,8 +1073,8 @@ public class XenServerStorageProcessor implements StorageProcessor { try { - final List parameters = new ArrayList( - serializeProperties(s3, S3Utils.ClientOptions.class)); + final List parameters = flattenProperties(s3, + S3Utils.ClientOptions.class); parameters.addAll(Arrays.asList("operation", "put", "directory", dir, "filename", filename, "iSCSIFlag", iSCSIFlag.toString(), "bucket", s3.getBucketName(), "key", key)); diff --git a/scripts/vm/hypervisor/xenserver/s3xen b/scripts/vm/hypervisor/xenserver/s3xen index 1348e483c69..4ee7c11402b 100644 --- a/scripts/vm/hypervisor/xenserver/s3xen +++ b/scripts/vm/hypervisor/xenserver/s3xen @@ -306,7 +306,7 @@ def parseArguments(args): # the com.cloud.utils.S3Utils#ClientOptions interface client = S3Client( args['accessKey'], args['secretKey'], args['endPoint'], - args['isHttps'], args['connectionTimeout'], args['socketTimeout']) + args['https'], args['connectionTimeout'], args['socketTimeout']) operation = args['operation'] bucket = args['bucket'] diff --git a/utils/src/com/cloud/utils/ReflectUtil.java b/utils/src/com/cloud/utils/ReflectUtil.java index e5a890a28e2..c123d299cd0 100755 --- a/utils/src/com/cloud/utils/ReflectUtil.java +++ b/utils/src/com/cloud/utils/ReflectUtil.java @@ -16,8 +16,17 @@ // under the License. package com.cloud.utils; +import com.cloud.utils.exception.CloudRuntimeException; +import com.google.common.collect.ImmutableSet; +import org.apache.log4j.Logger; +import org.reflections.Reflections; + +import java.beans.BeanInfo; +import java.beans.IntrospectionException; +import java.beans.PropertyDescriptor; import java.lang.annotation.Annotation; import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.util.ArrayList; import java.util.Collections; @@ -25,10 +34,14 @@ import java.util.HashSet; import java.util.List; import java.util.Set; -import com.cloud.utils.exception.CloudRuntimeException; -import org.reflections.Reflections; +import static java.beans.Introspector.getBeanInfo; +import static java.util.Collections.emptyList; +import static java.util.Collections.unmodifiableList; public class ReflectUtil { + + private static final Logger s_logger = Logger.getLogger(ReflectUtil.class); + public static Pair, Field> getAnyField(Class clazz, String fieldName) { try { return new Pair, Field>(clazz, clazz.getDeclaredField(fieldName)); @@ -129,4 +142,75 @@ public class ReflectUtil { return fields; } + public static List flattenProperties(final Object target, + final Class clazz) { + return flattenPropeties(target, clazz, "class"); + } + + public static List flattenPropeties(final Object target, + final Class clazz, + final String... + excludedProperties) { + return flattenProperties(target, clazz, + ImmutableSet.copyOf(excludedProperties)); + } + + private static List flattenProperties(final Object target, + final Class clazz, + final ImmutableSet + excludedProperties) { + + assert clazz != null; + + if (target == null) { + return emptyList(); + } + + assert clazz.isAssignableFrom(target.getClass()); + + try { + + final BeanInfo beanInfo = getBeanInfo(clazz); + final PropertyDescriptor[] descriptors = beanInfo + .getPropertyDescriptors(); + + final List serializedProperties = new ArrayList(); + for (final PropertyDescriptor descriptor : descriptors) { + + if (excludedProperties.contains(descriptor.getName())) { + continue; + } + + serializedProperties.add(descriptor.getName()); + final Object value = descriptor.getReadMethod().invoke(target); + serializedProperties.add(value != null ? value.toString() + : "null"); + + } + + return unmodifiableList(serializedProperties); + + } catch (IntrospectionException e) { + s_logger.warn( + "Ignored IntrospectionException when serializing class " + + target.getClass().getCanonicalName(), e); + } catch (IllegalArgumentException e) { + s_logger.warn( + "Ignored IllegalArgumentException when serializing class " + + target.getClass().getCanonicalName(), e); + } catch (IllegalAccessException e) { + s_logger.warn( + "Ignored IllegalAccessException when serializing class " + + target.getClass().getCanonicalName(), e); + } catch (InvocationTargetException e) { + s_logger.warn( + "Ignored InvocationTargetException when serializing class " + + target.getClass().getCanonicalName(), e); + } + + return emptyList(); + + } + + } diff --git a/utils/test/com/cloud/utils/ReflectUtilTest.java b/utils/test/com/cloud/utils/ReflectUtilTest.java new file mode 100644 index 00000000000..4c8eb506247 --- /dev/null +++ b/utils/test/com/cloud/utils/ReflectUtilTest.java @@ -0,0 +1,91 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License.package com.cloud.utils; +package com.cloud.utils; + +import java.util.List; +import org.junit.Test; + +import static com.cloud.utils.ReflectUtil.flattenProperties; +import static com.google.common.collect.Lists.newArrayList; +import static java.lang.Boolean.TRUE; +import static java.util.Collections.emptyList; +import static org.junit.Assert.assertEquals; + +public final class ReflectUtilTest { + + @Test + public void testFlattenNonNullProperties() throws Exception { + + final List expectedResult = newArrayList("booleanProperty", + TRUE.toString(), "intProperty", "1", + "stringProperty", "foo"); + + final Bean bean = new Bean(1, true, "foo"); + + assertEquals(expectedResult, flattenProperties(bean, Bean.class)); + + } + + @Test + public void testFlattenNullProperties() throws Exception { + + final List expectedResult = newArrayList("booleanProperty", + TRUE.toString(), "intProperty", "1", + "stringProperty", "null"); + + final Bean bean = new Bean(1, true, null); + + assertEquals(expectedResult, flattenProperties(bean, Bean.class)); + + } + + @Test + public void testFlattenPropertiesNullTarget() throws Exception { + assertEquals(emptyList(), flattenProperties(null, Bean.class)); + } + + public static final class Bean { + + private final int intProperty; + private final boolean booleanProperty; + private final String stringProperty; + + private Bean(final int intProperty, final boolean booleanProperty, + final String stringProperty) { + + super(); + + this.intProperty = intProperty; + this.booleanProperty = booleanProperty; + this.stringProperty = stringProperty; + + } + + public int getIntProperty() { + return intProperty; + } + + public boolean isBooleanProperty() { + return booleanProperty; + } + + public String getStringProperty() { + return stringProperty; + } + + } +} From 9cd4e089a5798f422961940efbf8ae33ed906b87 Mon Sep 17 00:00:00 2001 From: John Burwell Date: Thu, 1 Aug 2013 00:11:40 -0400 Subject: [PATCH 026/763] - CLOUDSTACK-3229: Fixes a think-o in the handling of unmodifiable collections and adds a rail in s3xen to raise an exception if the file being put does not exist --- .../cloud/hypervisor/xen/resource/CitrixResourceBase.java | 5 +++-- .../hypervisor/xen/resource/XenServerStorageProcessor.java | 5 +++-- scripts/vm/hypervisor/xenserver/s3xen | 6 +++++- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java b/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java index af408d8fb69..5bdf88e0b70 100644 --- a/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java +++ b/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java @@ -290,6 +290,7 @@ import java.util.Set; import java.util.UUID; import static com.cloud.utils.ReflectUtil.flattenProperties; +import static com.google.common.collect.Lists.newArrayList; /** * CitrixResourceBase encapsulates the calls to the XenServer Xapi process @@ -7440,8 +7441,8 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe try { - final List parameters = flattenProperties(s3, - S3Utils.ClientOptions.class); + final List parameters = newArrayList(flattenProperties(s3, + S3Utils.ClientOptions.class)); parameters.addAll(Arrays.asList("operation", "put", "directory", dir, "filename", filename, "iSCSIFlag", iSCSIFlag.toString(), "bucket", s3.getBucketName(), "key", key)); diff --git a/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/XenServerStorageProcessor.java b/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/XenServerStorageProcessor.java index 68a07c07ff1..9cd0b33b532 100644 --- a/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/XenServerStorageProcessor.java +++ b/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/XenServerStorageProcessor.java @@ -81,6 +81,7 @@ import java.util.Set; import java.util.UUID; import static com.cloud.utils.ReflectUtil.flattenProperties; +import static com.google.common.collect.Lists.newArrayList; public class XenServerStorageProcessor implements StorageProcessor { private static final Logger s_logger = Logger.getLogger(XenServerStorageProcessor.class); @@ -1073,8 +1074,8 @@ public class XenServerStorageProcessor implements StorageProcessor { try { - final List parameters = flattenProperties(s3, - S3Utils.ClientOptions.class); + final List parameters = newArrayList(flattenProperties(s3, + S3Utils.ClientOptions.class)); parameters.addAll(Arrays.asList("operation", "put", "directory", dir, "filename", filename, "iSCSIFlag", iSCSIFlag.toString(), "bucket", s3.getBucketName(), "key", key)); diff --git a/scripts/vm/hypervisor/xenserver/s3xen b/scripts/vm/hypervisor/xenserver/s3xen index 4ee7c11402b..8c1d3d8136e 100644 --- a/scripts/vm/hypervisor/xenserver/s3xen +++ b/scripts/vm/hypervisor/xenserver/s3xen @@ -179,7 +179,7 @@ class S3Client(object): max_error_retry, self.DEFAULT_MAX_ERROR_RETRY) def build_canocialized_resource(self, bucket, key): - return "/" + join([bucket, key], '/') + return "/" + join([bucket, key], "/") def noop_send_body(connection): pass @@ -256,6 +256,10 @@ class S3Client(object): def put(self, bucket, key, src_filename): + if not os.path.isfile(src_filename): + raise Exception( + "Attempt to put " + src_filename + " that does not exist.") + headers = { self.HEADER_CONTENT_MD5: compute_md5(src_filename), self.HEADER_CONTENT_TYPE: 'application/octet-stream', From 24af11a1c8e435034e5a31354f65c0ddc7312a12 Mon Sep 17 00:00:00 2001 From: Mice Xia Date: Thu, 1 Aug 2013 12:44:59 +0800 Subject: [PATCH 027/763] fix UI display bug for vmsnapshot, detailView of vmsnapshot always show the same one --- ui/scripts/vm_snapshots.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/scripts/vm_snapshots.js b/ui/scripts/vm_snapshots.js index d7b312cd713..cb780560d2b 100644 --- a/ui/scripts/vm_snapshots.js +++ b/ui/scripts/vm_snapshots.js @@ -111,7 +111,7 @@ }, dataProvider: function(args) { $.ajax({ - url: createURL("listVMSnapshot&listAll=true&id=" + args.context.vmsnapshots[0].id), + url: createURL("listVMSnapshot&listAll=true&vmsnapshotid=" + args.context.vmsnapshots[0].id), dataType: "json", async: true, success: function(json) { From 4271e60b703f6d0f9c64eeb61f25e5daee4313fb Mon Sep 17 00:00:00 2001 From: Sheng Yang Date: Wed, 31 Jul 2013 21:43:39 -0700 Subject: [PATCH 028/763] Automation: Fix test_vpc_network_lbrules.py --- .../component/test_vpc_network_lbrules.py | 120 ++++++++---------- 1 file changed, 51 insertions(+), 69 deletions(-) diff --git a/test/integration/component/test_vpc_network_lbrules.py b/test/integration/component/test_vpc_network_lbrules.py index a650cbc2f96..385c97ab6ae 100644 --- a/test/integration/component/test_vpc_network_lbrules.py +++ b/test/integration/component/test_vpc_network_lbrules.py @@ -43,6 +43,7 @@ from marvin.integration.lib.common import (get_domain, cleanup_resources, list_routers) import socket +import time class Services: """Test VPC network services Load Balancing Rules Test data @@ -126,10 +127,10 @@ class Services: "alg": "leastconn", # Algorithm used for load balancing "privateport": 22, - "publicport": 2222, + "publicport": 22, "openfirewall": False, "startport": 22, - "endport": 2222, + "endport": 22, "protocol": "TCP", "cidrlist": '0.0.0.0/0', }, @@ -138,10 +139,10 @@ class Services: "alg": "leastconn", # Algorithm used for load balancing "privateport": 80, - "publicport": 8888, + "publicport": 80, "openfirewall": False, "startport": 80, - "endport": 8888, + "endport": 80, "protocol": "TCP", "cidrlist": '0.0.0.0/0', }, @@ -197,7 +198,8 @@ class TestVPCNetworkLBRules(cloudstackTestCase): cls.services["ostype"] ) cls.services["virtual_machine"]["zoneid"] = cls.zone.id - cls.services["virtual_machine"]["template"] = cls.template.id + #cls.services["virtual_machine"]["template"] = cls.template.id + cls.services["virtual_machine"]["template"] = "719b88ea-5b54-4317-8bb3-d6b0b12da89c" cls.service_offering = ServiceOffering.create( cls.api_client, @@ -488,6 +490,14 @@ class TestVPCNetworkLBRules(cloudstackTestCase): ) self.debug("Adding virtual machines %s and %s to LB rule" % (vmarray[0], vmarray[1])) lb_rule.assign(self.apiclient, vmarray) + + self.debug("Adding NetworkACl rules to make NAT rule accessible") + nwacl_nat = NetworkACL.create(self.apiclient, + objservices, + networkid=network.id, + traffictype='Ingress' + ) + self.debug('nwacl_nat=%s' % nwacl_nat.__dict__) return lb_rule def create_egress_Internet_Rule(self, network): @@ -514,8 +524,9 @@ class TestVPCNetworkLBRules(cloudstackTestCase): # 5. Deploy vm1 and vm2 in network1. # 6. Deploy vm3 and vm4 in network2. # 7. Use the Create LB rule for vm1 and vm2 in network1. - # 8. Use the Create LB rule for vm3 amd vm4 in network2. - # 11. List LB rule + # 8. Use the Create LB rule for vm3 amd vm4 in network2, should fail + # because it's no_lb offering + # 9. List LB rule network_1 = self.create_Network(self.services["network_offering"]) network_2 = self.create_Network(self.services["network_offering_no_lb"], '10.1.2.1') @@ -525,9 +536,11 @@ class TestVPCNetworkLBRules(cloudstackTestCase): vm_3 = self.create_VM_in_Network(network_2) vm_4 = self.create_VM_in_Network(network_2) public_ip_1 = self.acquire_Public_IP(network_1) - lb_rule1 = self.create_LB_Rule(public_ip_1, network_1, [vm_1, vm_2]) + lb_rule1 = self.create_LB_Rule(public_ip_1, network_1, [vm_1, vm_2]) # public_ip_2 = self.acquire_Public_IP(network_2) - lb_rule2 = self.create_LB_Rule(public_ip_2, network_2, [vm_3, vm_4]) + with self.assertRaises(Exception): + self.create_LB_Rule(public_ip_2, network_2, [vm_3, vm_4]) + lb_rules = LoadBalancerRule.list(self.apiclient, id=lb_rule1.id, listall=True @@ -536,49 +549,6 @@ class TestVPCNetworkLBRules(cloudstackTestCase): None, "Failed to list the LB Rule" ) - lb_rules = LoadBalancerRule.list(self.apiclient, - id=lb_rule2.id, - listall=True - ) - self.failIfEqual(lb_rules, - None, - "Failed to list the LB Rule" - ) - return - - @attr(tags=["advanced", "intervlan"]) - def test_02_VPC_LBRulesAndVMListing(self): - """ Test case no 211 and 228: List only VMs suitable for the Virtual Network on VPC for LB Rule - """ - - # Validate the following - # 1. Create a VPC with cidr - 10.1.1.1/16 - # 2. Create a Network offering - NO1 with all supported services - # 3. Add network1(10.1.1.1/24) using N01 to this VPC. - # 4. Add network2(10.1.2.1/24) using N01 to this VPC. - # 5. Deploy vm1 and vm2 in network1 on primary host. - # 6. Deploy vm3 and vm4 in network2 on secondary host. - # 7. Use the Create LB rule for vm1 and vm2 in network1. - # 9. List LB rule for network1 list vms on network1 for selection of LB rule. - - network_1 = self.create_Network(self.services["network_offering"]) - network_2 = self.create_Network(self.services["network_offering_no_lb"], '10.1.2.1') - vm_1 = self.create_VM_in_Network(network_1) - vm_2 = self.create_VM_in_Network(network_1) - vm_3 = self.create_VM_in_Network(network_2) - self.debug('vm_3=%s' % vm_3.id) - vm_4 = self.create_VM_in_Network(network_2) - self.debug('vm_4=%s' % vm_4.id) - public_ip_1 = self.acquire_Public_IP(network_1) - lb_rule = self.create_LB_Rule(public_ip_1, network_1, [vm_1, vm_2]) - lb_rules = LoadBalancerRule.list(self.apiclient, - id=lb_rule.id, - listall=True - ) - self.failIfEqual(lb_rules, - None, - "Failed to list the LB Rule" - ) vms = VirtualMachine.list(self.apiclient, networkid=network_1.id, listall=True @@ -590,7 +560,7 @@ class TestVPCNetworkLBRules(cloudstackTestCase): return @attr(tags=["advanced", "intervlan"]) - def test_03_VPC_CreateLBRuleInMultipleNetworks(self): + def test_02_VPC_CreateLBRuleInMultipleNetworks(self): """ Test Create LB rules for 1 network which is part of a two/multiple virtual networks of a VPC using a new Public IP Address available with the VPC when the Virtual Router is in Running State """ @@ -599,17 +569,15 @@ class TestVPCNetworkLBRules(cloudstackTestCase): # 1. Create a VPC with cidr - 10.1.1.1/16 # 2. Create a Network offering - NO1 with all supported services # 3. Add network1(10.1.1.1/24) using N01 to this VPC. - # 4. Add network2(10.1.2.1/24) using N01 to this VPC. - # 5. Deploy vm1, vm2 and vm3 in network1 on primary host. - # 7. Use the Create LB rule for vm1 and vm2 in network1. - # 8. Add vm3 to LB rule. - # 9. wget a file and check for LB rule. + # 4. Deploy vm1, vm2 and vm3 in network1 on primary host. + # 5. Use the Create LB rule for vm1 and vm2 in network1. + # 6. Add vm3 to LB rule. + # 7. wget a file and check for LB rule. network_1 = self.create_Network(self.services["network_offering"]) - network_2 = self.create_Network(self.services["network_offering_no_lb"], '10.1.2.1') vm_1 = self.create_VM_in_Network(network_1) vm_2 = self.create_VM_in_Network(network_1) - vm_3 = self.create_VM_in_Network(network_2) + vm_3 = self.create_VM_in_Network(network_1) public_ip_1 = self.acquire_Public_IP(network_1) lb_rule = self.create_LB_Rule(public_ip_1, network_1, [vm_1, vm_2], self.services["lbrule_http"]) lb_rule.assign(self.apiclient, [vm_3]) @@ -617,7 +585,7 @@ class TestVPCNetworkLBRules(cloudstackTestCase): return @attr(tags=["advanced", "intervlan"]) - def test_04_VPC_CreateLBRuleInMultipleNetworksVRStoppedState(self): + def test_03_VPC_CreateLBRuleInMultipleNetworksVRStoppedState(self): """ Test case no 222 : Create LB rules for a two/multiple virtual networks of a VPC using a new Public IP Address available with the VPC when the Virtual Router is in Stopped State """ @@ -636,10 +604,19 @@ class TestVPCNetworkLBRules(cloudstackTestCase): network_2 = self.create_Network(self.services["network_offering_no_lb"], '10.1.2.1') vm_1 = self.create_VM_in_Network(network_1) vm_2 = self.create_VM_in_Network(network_1) - vm_3 = self.create_VM_in_Network(network_2) + vm_3 = self.create_VM_in_Network(network_1) + + # wait until VM is up before stop the VR + time.sleep(120) + + router = self.stop_VPC_VRouter() + public_ip_1 = self.acquire_Public_IP(network_1) lb_rule = self.create_LB_Rule(public_ip_1, network_1, [vm_1, vm_2], self.services["lbrule_http"]) lb_rule.assign(self.apiclient, [vm_3]) + + self.start_VPC_VRouter(router) + self.check_wget_from_vm(vm_1, public_ip_1, testnegative=False) return @@ -695,10 +672,16 @@ class TestVPCNetworkLBRules(cloudstackTestCase): vm_1 = self.create_VM_in_Network(network_1) vm_2 = self.create_VM_in_Network(network_1) vm_3 = self.create_VM_in_Network(network_1) + # wait until VM is up before stop the VR + time.sleep(120) + + router = self.stop_VPC_VRouter() public_ip_1 = self.acquire_Public_IP(network_1) lb_rule_http = self.create_LB_Rule(public_ip_1, network_1, [vm_1, vm_2, vm_3], self.services["lbrule_http"]) lb_rule_nat = self.create_LB_Rule(public_ip_1, network_1, [vm_1, vm_2, vm_3]) self.debug('lb_rule_http=%s' % lb_rule_http.__dict__) + self.start_VPC_VRouter(router) + self.check_wget_from_vm(vm_1, public_ip_1, testnegative=False) self.check_ssh_into_vm(vm_1, public_ip_1, testnegative=False) lb_rule_nat.delete(self.apiclient) @@ -799,11 +782,11 @@ class TestVPCNetworkLBRules(cloudstackTestCase): vm_3 = self.create_VM_in_Network(network_2) vm_4 = self.create_VM_in_Network(network_2) public_ip_1 = self.acquire_Public_IP(network_1) - lb_rule = self.create_LB_Rule(public_ip_1, network_1, [vm_1, vm_2]) + lb_rule = self.create_LB_Rule(public_ip_1, network_1, [vm_1, vm_2], self.services["lbrule_http"]) self.debug('lb_rule=%s' % lb_rule.__dict__) self.check_wget_from_vm(vm_1, public_ip_1, testnegative=False) try: - lb_rule = self.create_LB_Rule(public_ip_1, network_1, [vm_3, vm_4]) + lb_rule = self.create_LB_Rule(public_ip_1, network_1, [vm_3, vm_4], self.services["lbrule_http"]) self.fail('Successfully created LB rule for vm_3, vm_4 in network1') except: self.debug('Failed to Create LB rule vm_3 and vm_4') @@ -836,11 +819,11 @@ class TestVPCNetworkLBRules(cloudstackTestCase): vm_4 = self.create_VM_in_Network(network_3) self.debug('vm_4=%s' % vm_4.id) public_ip_1 = self.acquire_Public_IP(network_1) - lb_rule = self.create_LB_Rule(public_ip_1, network_1, [vm_1, vm_2]) + lb_rule = self.create_LB_Rule(public_ip_1, network_1, [vm_1, vm_2], self.services["lbrule_http"]) self.debug('lb_rule=%s' % lb_rule.__dict__) self.check_wget_from_vm(vm_1, public_ip_1, testnegative=True) try: - lb_rule = self.create_LB_Rule(public_ip_1, network_2, [vm_3, vm_4]) + lb_rule = self.create_LB_Rule(public_ip_1, network_2, [vm_3, vm_4], self.services["lbrule_http"]) self.fail('Successfully created LB rule for vm_3, vm_4 in network2') except: self.debug('Failed to Create LB rule vm_3 and vm_4 in network2') @@ -874,11 +857,11 @@ class TestVPCNetworkLBRules(cloudstackTestCase): vm_4 = self.create_VM_in_Network(network_2) self.debug('vm_4=%s' % vm_4.id) public_ip_1 = self.acquire_Public_IP(network_1) - lb_rule = self.create_LB_Rule(public_ip_1, network_1, [vm_1, vm_2]) + lb_rule = self.create_LB_Rule(public_ip_1, network_1, [vm_1, vm_2], self.services["lbrule_http"]) self.debug('lb_rule=%s' % lb_rule.__dict__) self.check_wget_from_vm(vm_1, public_ip_1, testnegative=False) try: - lb_rule = self.create_LB_Rule(public_ip_1, network_1, [vm_3, vm_1]) + lb_rule = self.create_LB_Rule(public_ip_1, network_1, [vm_3, vm_1], self.services["lbrule_http"]) self.fail('Successfully created LB rule for vm_3, vm_1 in network1') except: self.debug('Failed to Create LB rule vm_3 and vm_1') @@ -964,7 +947,6 @@ class TestVPCNetworkLBRules(cloudstackTestCase): vm_2 = self.create_VM_in_Network(network_1) public_ip_1 = self.acquire_Public_IP(network_1) self.create_StaticNatRule_For_VM(vm_1, public_ip_1, network_1) - self.check_wget_from_vm(vm_1, public_ip_1, testnegative=True) try: lb_rule = self.create_LB_Rule(public_ip_1, network_1, [vm_2, vm_1]) self.fail('Successfully created LB rule for vm_2, vm_1 in network1 %s=' % lb_rule.__dict__) From 22b0375df20ae8aebb0ec9752b786353ebb87ca6 Mon Sep 17 00:00:00 2001 From: Sheng Yang Date: Wed, 31 Jul 2013 22:41:02 -0700 Subject: [PATCH 029/763] Automation: Removing the template's UUID My bad... --- test/integration/component/test_vpc_network_lbrules.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/integration/component/test_vpc_network_lbrules.py b/test/integration/component/test_vpc_network_lbrules.py index 385c97ab6ae..074d474586b 100644 --- a/test/integration/component/test_vpc_network_lbrules.py +++ b/test/integration/component/test_vpc_network_lbrules.py @@ -198,8 +198,7 @@ class TestVPCNetworkLBRules(cloudstackTestCase): cls.services["ostype"] ) cls.services["virtual_machine"]["zoneid"] = cls.zone.id - #cls.services["virtual_machine"]["template"] = cls.template.id - cls.services["virtual_machine"]["template"] = "719b88ea-5b54-4317-8bb3-d6b0b12da89c" + cls.services["virtual_machine"]["template"] = cls.template.id cls.service_offering = ServiceOffering.create( cls.api_client, From 5804aa71ddae4433898ead7c0ae2cfe7385637fd Mon Sep 17 00:00:00 2001 From: Radhika PC Date: Thu, 1 Aug 2013 11:10:20 +0530 Subject: [PATCH 030/763] CLOUDSTACK-1567 --- docs/en-US/delete-event-alerts.xml | 47 +++++++++++++++++++++++------- 1 file changed, 36 insertions(+), 11 deletions(-) diff --git a/docs/en-US/delete-event-alerts.xml b/docs/en-US/delete-event-alerts.xml index 5958b721940..c0d56719818 100644 --- a/docs/en-US/delete-event-alerts.xml +++ b/docs/en-US/delete-event-alerts.xml @@ -44,19 +44,44 @@ - Archived alerts or events cannot be viewed in the UI, or by using the API. They are + Archived alerts or events cannot be viewed in the UI or by using the API. They are maintained in the database for auditing or compliance purposes. - +
Permissions Consider the following: - - - - The root admin can delete or archive one or multiple alerts or events. - - - The domain admin or end user can delete or archive one or multiple events. - - + + + The root admin can delete or archive one or multiple alerts or events. + + + The domain admin or end user can delete or archive one or multiple events. + + +
+
+ Procedure + + + Log in as administrator to the &PRODUCT; UI. + + + In the left navigation, click Events. + + + Perform either of the following: + + + To archive events, click Archive Events, and specify event type and date. + + + To archive events, click Delete Events, and specify event type and date. + + + + + Click OK. + + +
From af631c3e0e0e0eaa09558dba5302014cd798c6ef Mon Sep 17 00:00:00 2001 From: John Burwell Date: Thu, 1 Aug 2013 01:41:24 -0400 Subject: [PATCH 031/763] - CLOUDSTACK-3229: Works around Introspector bug that does not recognize Boolean typed accessor methods by manualy populating the https value. A generic implementation will be investigated when time permits. --- .../cloud/hypervisor/xen/resource/CitrixResourceBase.java | 6 +++++- .../hypervisor/xen/resource/XenServerStorageProcessor.java | 7 ++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java b/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java index 5bdf88e0b70..f22a8621b82 100644 --- a/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java +++ b/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java @@ -7443,9 +7443,13 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe final List parameters = newArrayList(flattenProperties(s3, S3Utils.ClientOptions.class)); + // https workaround for Introspector bug that does not + // recognize Boolean accessor methods ... parameters.addAll(Arrays.asList("operation", "put", "directory", dir, "filename", filename, "iSCSIFlag", - iSCSIFlag.toString(), "bucket", s3.getBucketName(), "key", key)); + iSCSIFlag.toString(), "bucket", s3.getBucketName(), + "key", key, "https", s3.isHttps() != null ? s3.isHttps().toString() + : "null")); final String result = callHostPluginAsync(connection, "s3xen", "s3", wait, parameters.toArray(new String[parameters.size()])); diff --git a/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/XenServerStorageProcessor.java b/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/XenServerStorageProcessor.java index 9cd0b33b532..91909521332 100644 --- a/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/XenServerStorageProcessor.java +++ b/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/XenServerStorageProcessor.java @@ -68,6 +68,7 @@ import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; import org.apache.cloudstack.storage.to.SnapshotObjectTO; import org.apache.cloudstack.storage.to.TemplateObjectTO; import org.apache.cloudstack.storage.to.VolumeObjectTO; +import org.apache.cloudstack.storage.to.VolumeObjectTO; import org.apache.log4j.Logger; import org.apache.xmlrpc.XmlRpcException; @@ -1076,9 +1077,13 @@ public class XenServerStorageProcessor implements StorageProcessor { final List parameters = newArrayList(flattenProperties(s3, S3Utils.ClientOptions.class)); + // https workaround for Introspector bug that does not + // recognize Boolean accessor methods ... parameters.addAll(Arrays.asList("operation", "put", "directory", dir, "filename", filename, "iSCSIFlag", - iSCSIFlag.toString(), "bucket", s3.getBucketName(), "key", key)); + iSCSIFlag.toString(), "bucket", s3.getBucketName(), + "key", key, "https", s3.isHttps() != null ? s3.isHttps().toString() + : "null")); final String result = hypervisorResource.callHostPluginAsync(connection, "s3xen", "s3", wait, parameters.toArray(new String[parameters.size()])); From 5e3b7516338ad77011ab9d04a3d1ce251dac2307 Mon Sep 17 00:00:00 2001 From: Sheng Yang Date: Wed, 31 Jul 2013 23:13:24 -0700 Subject: [PATCH 032/763] Automation: More fix for test_vpc_network_lbrules.py --- test/integration/component/test_vpc_network_lbrules.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/test/integration/component/test_vpc_network_lbrules.py b/test/integration/component/test_vpc_network_lbrules.py index 074d474586b..5aa626ab8a8 100644 --- a/test/integration/component/test_vpc_network_lbrules.py +++ b/test/integration/component/test_vpc_network_lbrules.py @@ -975,14 +975,10 @@ class TestVPCNetworkLBRules(cloudstackTestCase): public_ip_1 = self.acquire_Public_IP(network_1) lb_rule = self.create_LB_Rule(public_ip_1, network_1, [vm_2, vm_1]) public_ip_1.delete(self.apiclient) - lb_rules = LoadBalancerRule.list(self.apiclient, + + with self.assertRaises(Exception): + lb_rules = LoadBalancerRule.list(self.apiclient, id=lb_rule.id, listall=True ) - self.assertEqual(lb_rules, - None, - "Failed LB rule is present on the VR" - ) - - return From 1cc98ee7aeab7b9dff46da9c9a592c2e56bb0f52 Mon Sep 17 00:00:00 2001 From: John Burwell Date: Thu, 1 Aug 2013 02:26:00 -0400 Subject: [PATCH 033/763] - CLODSTACK-3229: Adds missing is_blank function to s3xen --- scripts/vm/hypervisor/xenserver/s3xen | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/vm/hypervisor/xenserver/s3xen b/scripts/vm/hypervisor/xenserver/s3xen index 8c1d3d8136e..44a30efd4b3 100644 --- a/scripts/vm/hypervisor/xenserver/s3xen +++ b/scripts/vm/hypervisor/xenserver/s3xen @@ -78,6 +78,11 @@ def optional_str_value(value, default): return default +def is_blank(value): + + return not is_not_blank(value) + + def is_not_blank(value): if to_none(value) is None or not isinstance(value, basestring): From 952c9daa20bae644f1a9bb2236b031dce747926a Mon Sep 17 00:00:00 2001 From: John Burwell Date: Thu, 1 Aug 2013 02:44:17 -0400 Subject: [PATCH 034/763] - CLOUDSTACK-3229: Removes String format method call that is not supported in Python 2.4 --- scripts/vm/hypervisor/xenserver/s3xen | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/vm/hypervisor/xenserver/s3xen b/scripts/vm/hypervisor/xenserver/s3xen index 44a30efd4b3..a87e6de5dbe 100644 --- a/scripts/vm/hypervisor/xenserver/s3xen +++ b/scripts/vm/hypervisor/xenserver/s3xen @@ -352,7 +352,7 @@ def s3(session, args): client.delete(bucket, key, filename) else: raise RuntimeError( - "S3 plugin does not support operation {0}.".format(operation)) + "S3 plugin does not support operation " + operation)) return 'true' From 381318928348289f5dacd9d482302a9be561f6bd Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Thu, 1 Aug 2013 12:15:18 +0530 Subject: [PATCH 035/763] CLOUDSTACK-3042: handle System VM Scaling up of memory/CPU based on the presence of tools in the template Signed off by : Nitin Mehta --- server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java | 1 + .../cloud/network/router/VirtualNetworkApplianceManagerImpl.java | 1 + .../com/cloud/storage/secondary/SecondaryStorageManagerImpl.java | 1 + 3 files changed, 3 insertions(+) diff --git a/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java b/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java index 5983aa7ad7c..b956f3ecb94 100755 --- a/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java +++ b/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java @@ -724,6 +724,7 @@ public class ConsoleProxyManagerImpl extends ManagerBase implements ConsoleProxy ConsoleProxyVO proxy = new ConsoleProxyVO(id, _serviceOffering.getId(), name, template.getId(), template.getHypervisorType(), template.getGuestOSId(), dataCenterId, systemAcct.getDomainId(), systemAcct.getId(), 0, _serviceOffering.getOfferHA()); + proxy.setDynamicallyScalable(template.isDynamicallyScalable()); try { proxy = _itMgr.allocate(proxy, template, _serviceOffering, networks, plan, null, systemAcct); } catch (InsufficientCapacityException e) { diff --git a/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java b/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java index 3ee8a1c61d6..7a1a05b8201 100755 --- a/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java +++ b/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java @@ -1648,6 +1648,7 @@ public class VirtualNetworkApplianceManagerImpl extends ManagerBase implements V VirtualMachineName.getRouterName(id, _instance), template.getId(), template.getHypervisorType(), template.getGuestOSId(), owner.getDomainId(), owner.getId(), isRedundant, 0, false, RedundantState.UNKNOWN, offerHA, false, vpcId); + router.setDynamicallyScalable(template.isDynamicallyScalable()); router.setRole(Role.VIRTUAL_ROUTER); router = _itMgr.allocate(router, template, routerOffering, networks, plan, null, owner); } catch (InsufficientCapacityException ex) { diff --git a/server/src/com/cloud/storage/secondary/SecondaryStorageManagerImpl.java b/server/src/com/cloud/storage/secondary/SecondaryStorageManagerImpl.java index 6859b0bb20d..81aa07263a0 100755 --- a/server/src/com/cloud/storage/secondary/SecondaryStorageManagerImpl.java +++ b/server/src/com/cloud/storage/secondary/SecondaryStorageManagerImpl.java @@ -578,6 +578,7 @@ public class SecondaryStorageManagerImpl extends ManagerBase implements Secondar SecondaryStorageVmVO secStorageVm = new SecondaryStorageVmVO(id, _serviceOffering.getId(), name, template.getId(), template.getHypervisorType(), template.getGuestOSId(), dataCenterId, systemAcct.getDomainId(), systemAcct.getId(), role, _serviceOffering.getOfferHA()); + secStorageVm.setDynamicallyScalable(template.isDynamicallyScalable()); try { secStorageVm = _itMgr.allocate(secStorageVm, template, _serviceOffering, networks, plan, null, systemAcct); } catch (InsufficientCapacityException e) { From fe85a9e6d827bba8eee3cddf86a6d309f680d83c Mon Sep 17 00:00:00 2001 From: John Burwell Date: Thu, 1 Aug 2013 02:53:05 -0400 Subject: [PATCH 036/763] - CLOUDSTACK-3229: Fixes a syntax error in s3xen --- scripts/vm/hypervisor/xenserver/s3xen | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/vm/hypervisor/xenserver/s3xen b/scripts/vm/hypervisor/xenserver/s3xen index a87e6de5dbe..015e7dfd23a 100644 --- a/scripts/vm/hypervisor/xenserver/s3xen +++ b/scripts/vm/hypervisor/xenserver/s3xen @@ -352,7 +352,7 @@ def s3(session, args): client.delete(bucket, key, filename) else: raise RuntimeError( - "S3 plugin does not support operation " + operation)) + "S3 plugin does not support operation " + operation) return 'true' From 42e950aa413d67f6993ff85218c36047b0cb3d9e Mon Sep 17 00:00:00 2001 From: John Burwell Date: Thu, 1 Aug 2013 03:12:18 -0400 Subject: [PATCH 037/763] - CLOUDSTACK-3229: Fixes string formatting argument mismatch --- scripts/vm/hypervisor/xenserver/s3xen | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/vm/hypervisor/xenserver/s3xen b/scripts/vm/hypervisor/xenserver/s3xen index 015e7dfd23a..454097b4458 100644 --- a/scripts/vm/hypervisor/xenserver/s3xen +++ b/scripts/vm/hypervisor/xenserver/s3xen @@ -109,7 +109,7 @@ def echo(fn): name = fn.__name__ log("enter %s ####" % name) res = fn(*v, **k) - log("exit %s with result %s" % name, res) + log("exit %s with result %s" % (name, res)) return res return wrapped From 1162b87f142e930b5b3518e2a3937981b2d47bad Mon Sep 17 00:00:00 2001 From: John Burwell Date: Thu, 1 Aug 2013 03:48:52 -0400 Subject: [PATCH 038/763] - CLOUDSTACK-3229: Attempt to adjust snapshot source path to new cache filesystem layout --- .../xen/resource/CitrixResourceBase.java | 16 ++++++++-------- .../xen/resource/XenServerStorageProcessor.java | 10 +++++----- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java b/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java index f22a8621b82..49433f86e89 100644 --- a/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java +++ b/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java @@ -7430,14 +7430,14 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe } private boolean backupSnapshotToS3(final Connection connection, - final S3TO s3, final String srUuid, final String snapshotUuid, - final Boolean iSCSIFlag, final int wait) { + final S3TO s3, final String srUuid, final String snapshotUuid, + final Boolean iSCSIFlag, final int wait) { final String filename = iSCSIFlag ? "VHD-" + snapshotUuid : snapshotUuid + ".vhd"; final String dir = (iSCSIFlag ? "/dev/VG_XenStorage-" : "/var/run/sr-mount/") + srUuid; - final String key = StringUtils.join("/", "snapshots", snapshotUuid); + final String key = String.format("/snapshots/%1$s", snapshotUuid); try { @@ -7445,11 +7445,11 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe S3Utils.ClientOptions.class)); // https workaround for Introspector bug that does not // recognize Boolean accessor methods ... - parameters.addAll(Arrays.asList("operation", "put", "directory", - dir, "filename", filename, "iSCSIFlag", - iSCSIFlag.toString(), "bucket", s3.getBucketName(), - "key", key, "https", s3.isHttps() != null ? s3.isHttps().toString() - : "null")); + parameters.addAll(Arrays.asList("operation", "put", "filename", + dir + "/" + filename, "iSCSIFlag", iSCSIFlag.toString(), + "bucket", s3.getBucketName(), "key", key, "https", + s3.isHttps() != null ? s3.isHttps().toString() + : "null")); final String result = callHostPluginAsync(connection, "s3xen", "s3", wait, parameters.toArray(new String[parameters.size()])); diff --git a/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/XenServerStorageProcessor.java b/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/XenServerStorageProcessor.java index 91909521332..634759deb7b 100644 --- a/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/XenServerStorageProcessor.java +++ b/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/XenServerStorageProcessor.java @@ -1064,14 +1064,14 @@ public class XenServerStorageProcessor implements StorageProcessor { } private boolean backupSnapshotToS3(final Connection connection, - final S3TO s3, final String srUuid, final String snapshotUuid, - final Boolean iSCSIFlag, final int wait) { + final S3TO s3, final String srUuid, final String snapshotUuid, + final Boolean iSCSIFlag, final int wait) { final String filename = iSCSIFlag ? "VHD-" + snapshotUuid : snapshotUuid + ".vhd"; final String dir = (iSCSIFlag ? "/dev/VG_XenStorage-" : "/var/run/sr-mount/") + srUuid; - final String key = StringUtils.join("/", "snapshots", snapshotUuid); + final String key = String.format("/snapshots/%1$s", snapshotUuid); try { @@ -1079,8 +1079,8 @@ public class XenServerStorageProcessor implements StorageProcessor { S3Utils.ClientOptions.class)); // https workaround for Introspector bug that does not // recognize Boolean accessor methods ... - parameters.addAll(Arrays.asList("operation", "put", "directory", - dir, "filename", filename, "iSCSIFlag", + parameters.addAll(Arrays.asList("operation", "put", "filename", + dir + "/" + filename, "iSCSIFlag", iSCSIFlag.toString(), "bucket", s3.getBucketName(), "key", key, "https", s3.isHttps() != null ? s3.isHttps().toString() : "null")); From 698253576a847dc70e82afc04c773a4eebd4018f Mon Sep 17 00:00:00 2001 From: Wido den Hollander Date: Thu, 18 Jul 2013 18:16:39 +0200 Subject: [PATCH 039/763] rbd: Enable snapshotting of RBD images Signed-off-by: Wido den Hollander --- plugins/hypervisors/kvm/pom.xml | 6 + .../resource/LibvirtComputingResource.java | 164 ++++++++++++++---- .../kvm/storage/LibvirtStorageAdaptor.java | 144 +++++++++++---- .../kvm/storage/LibvirtStoragePool.java | 2 +- pom.xml | 3 +- 5 files changed, 250 insertions(+), 69 deletions(-) diff --git a/plugins/hypervisors/kvm/pom.xml b/plugins/hypervisors/kvm/pom.xml index 1babe7cbf56..8128ee9ff9f 100644 --- a/plugins/hypervisors/kvm/pom.xml +++ b/plugins/hypervisors/kvm/pom.xml @@ -45,6 +45,12 @@ rados ${cs.rados-java.version} + + net.java.dev.jna + jna + provided + ${cs.jna.version} + install diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 56699afec40..7330fa7daf4 100755 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -21,6 +21,7 @@ import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileOutputStream; +import java.io.BufferedOutputStream; import java.io.FileReader; import java.io.IOException; import java.io.InputStream; @@ -242,6 +243,13 @@ import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachine.State; import com.cloud.vm.VirtualMachineName; +import com.ceph.rados.Rados; +import com.ceph.rados.RadosException; +import com.ceph.rados.IoCTX; +import com.ceph.rbd.Rbd; +import com.ceph.rbd.RbdImage; +import com.ceph.rbd.RbdException; + /** * LibvirtComputingResource execute requests on the computing/routing host using * the libvirt API @@ -1975,12 +1983,6 @@ ServerResource { cmd.getPool().getType(), cmd.getPool().getUuid()); - if (primaryPool.getType() == StoragePoolType.RBD) { - s_logger.debug("Snapshots are not supported on RBD volumes"); - return new ManageSnapshotAnswer(cmd, false, - "Snapshots are not supported on RBD volumes"); - } - KVMPhysicalDisk disk = primaryPool.getPhysicalDisk(cmd .getVolumePath()); if (state == DomainInfo.DomainState.VIR_DOMAIN_RUNNING @@ -2007,23 +2009,63 @@ ServerResource { vm.resume(); } } else { + /** + * For RBD we can't use libvirt to do our snapshotting or any Bash scripts. + * libvirt also wants to store the memory contents of the Virtual Machine, + * but that's not possible with RBD since there is no way to store the memory + * contents in RBD. + * + * So we rely on the Java bindings for RBD to create our snapshot + * + * This snapshot might not be 100% consistent due to writes still being in the + * memory of the Virtual Machine, but if the VM runs a kernel which supports + * barriers properly (>2.6.32) this won't be any different then pulling the power + * cord out of a running machine. + */ + if (primaryPool.getType() == StoragePoolType.RBD) { + try { + Rados r = new Rados(primaryPool.getAuthUserName()); + r.confSet("mon_host", primaryPool.getSourceHost() + ":" + primaryPool.getSourcePort()); + r.confSet("key", primaryPool.getAuthSecret()); + r.connect(); + s_logger.debug("Succesfully connected to Ceph cluster at " + r.confGet("mon_host")); - /* VM is not running, create a snapshot by ourself */ - final Script command = new Script(_manageSnapshotPath, - _cmdsTimeout, s_logger); - if (cmd.getCommandSwitch().equalsIgnoreCase( - ManageSnapshotCommand.CREATE_SNAPSHOT)) { - command.add("-c", disk.getPath()); + IoCTX io = r.ioCtxCreate(primaryPool.getSourceDir()); + Rbd rbd = new Rbd(io); + RbdImage image = rbd.open(disk.getName()); + + if (cmd.getCommandSwitch().equalsIgnoreCase( + ManageSnapshotCommand.CREATE_SNAPSHOT)) { + s_logger.debug("Attempting to create RBD snapshot " + disk.getName() + "@" + snapshotName); + image.snapCreate(snapshotName); + } else { + s_logger.debug("Attempting to remove RBD snapshot " + disk.getName() + "@" + snapshotName); + image.snapRemove(snapshotName); + } + + rbd.close(image); + r.ioCtxDestroy(io); + } catch (Exception e) { + s_logger.error("A RBD snapshot operation on " + disk.getName() + " failed. The error was: " + e.getMessage()); + } } else { - command.add("-d", snapshotPath); - } + /* VM is not running, create a snapshot by ourself */ + final Script command = new Script(_manageSnapshotPath, + _cmdsTimeout, s_logger); + if (cmd.getCommandSwitch().equalsIgnoreCase( + ManageSnapshotCommand.CREATE_SNAPSHOT)) { + command.add("-c", disk.getPath()); + } else { + command.add("-d", snapshotPath); + } - command.add("-n", snapshotName); - String result = command.execute(); - if (result != null) { - s_logger.debug("Failed to manage snapshot: " + result); - return new ManageSnapshotAnswer(cmd, false, - "Failed to manage snapshot: " + result); + command.add("-n", snapshotName); + String result = command.execute(); + if (result != null) { + s_logger.debug("Failed to manage snapshot: " + result); + return new ManageSnapshotAnswer(cmd, false, + "Failed to manage snapshot: " + result); + } } } return new ManageSnapshotAnswer(cmd, cmd.getSnapshotId(), @@ -2065,16 +2107,74 @@ ServerResource { cmd.getPrimaryStoragePoolNameLabel()); KVMPhysicalDisk snapshotDisk = primaryPool.getPhysicalDisk(cmd .getVolumePath()); - Script command = new Script(_manageSnapshotPath, _cmdsTimeout, - s_logger); - command.add("-b", snapshotDisk.getPath()); - command.add("-n", snapshotName); - command.add("-p", snapshotDestPath); - command.add("-t", snapshotName); - String result = command.execute(); - if (result != null) { - s_logger.debug("Failed to backup snaptshot: " + result); - return new BackupSnapshotAnswer(cmd, false, result, null, true); + + /** + * RBD snapshots can't be copied using qemu-img, so we have to use + * the Java bindings for librbd here. + * + * These bindings will read the snapshot and write the contents to + * the secondary storage directly + * + * It will stop doing so if the amount of time spend is longer then + * cmds.timeout + */ + if (primaryPool.getType() == StoragePoolType.RBD) { + try { + Rados r = new Rados(primaryPool.getAuthUserName()); + r.confSet("mon_host", primaryPool.getSourceHost() + ":" + primaryPool.getSourcePort()); + r.confSet("key", primaryPool.getAuthSecret()); + r.connect(); + s_logger.debug("Succesfully connected to Ceph cluster at " + r.confGet("mon_host")); + + IoCTX io = r.ioCtxCreate(primaryPool.getSourceDir()); + Rbd rbd = new Rbd(io); + RbdImage image = rbd.open(snapshotDisk.getName(), snapshotName); + + long startTime = System.currentTimeMillis() / 1000; + + File fh = new File(snapshotDestPath); + BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(fh)); + int chunkSize = 4194304; + long offset = 0; + s_logger.debug("Backuping up RBD snapshot " + snapshotName + " to " + snapshotDestPath); + while(true) { + byte[] buf = new byte[chunkSize]; + + int bytes = image.read(offset, buf, chunkSize); + if (bytes <= 0) { + break; + } + bos.write(buf, 0, bytes); + offset += bytes; + } + s_logger.debug("Completed backing up RBD snapshot " + snapshotName + " to " + snapshotDestPath + ". Bytes written: " + offset); + bos.close(); + r.ioCtxDestroy(io); + } catch (RadosException e) { + s_logger.error("A RADOS operation failed. The error was: " + e.getMessage()); + return new BackupSnapshotAnswer(cmd, false, e.toString(), null, true); + } catch (RbdException e) { + s_logger.error("A RBD operation on " + snapshotDisk.getName() + " failed. The error was: " + e.getMessage()); + return new BackupSnapshotAnswer(cmd, false, e.toString(), null, true); + } catch (FileNotFoundException e) { + s_logger.error("Failed to open " + snapshotDestPath + ". The error was: " + e.getMessage()); + return new BackupSnapshotAnswer(cmd, false, e.toString(), null, true); + } catch (IOException e) { + s_logger.debug("An I/O error occured during a snapshot operation on " + snapshotDestPath); + return new BackupSnapshotAnswer(cmd, false, e.toString(), null, true); + } + } else { + Script command = new Script(_manageSnapshotPath, _cmdsTimeout, + s_logger); + command.add("-b", snapshotDisk.getPath()); + command.add("-n", snapshotName); + command.add("-p", snapshotDestPath); + command.add("-t", snapshotName); + String result = command.execute(); + if (result != null) { + s_logger.debug("Failed to backup snaptshot: " + result); + return new BackupSnapshotAnswer(cmd, false, result, null, true); + } } /* Delete the snapshot on primary */ @@ -2111,11 +2211,11 @@ ServerResource { vm.resume(); } } else { - command = new Script(_manageSnapshotPath, _cmdsTimeout, + Script command = new Script(_manageSnapshotPath, _cmdsTimeout, s_logger); command.add("-d", snapshotDisk.getPath()); command.add("-n", snapshotName); - result = command.execute(); + String result = command.execute(); if (result != null) { s_logger.debug("Failed to backup snapshot: " + result); return new BackupSnapshotAnswer(cmd, false, diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java index d743cd0754d..a9baa52d71c 100644 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java @@ -46,6 +46,7 @@ import com.ceph.rados.IoCTX; import com.ceph.rbd.Rbd; import com.ceph.rbd.RbdImage; import com.ceph.rbd.RbdException; +import com.ceph.rbd.jna.RbdSnapInfo; import com.cloud.agent.api.ManageSnapshotCommand; import com.cloud.hypervisor.kvm.resource.LibvirtConnection; @@ -73,6 +74,8 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { private String _manageSnapshotPath; private String rbdTemplateSnapName = "cloudstack-base-snap"; + private int rbdFeatures = (1<<0); /* Feature 1<<0 means layering in RBD format 2 */ + private int rbdOrder = 0; /* Order 0 means 4MB blocks (the default) */ public LibvirtStorageAdaptor(StorageLayer storage) { _storageLayer = storage; @@ -615,38 +618,116 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { StoragePool virtPool = libvirtPool.getPool(); LibvirtStorageVolumeDef.volFormat libvirtformat = null; + String volPath = null; + String volName = null; + long volAllocation = 0; + long volCapacity = 0; + + /** + * To have RBD function properly we want RBD images of format 2 + * libvirt currently defaults to format 1 + * + * For that reason we use the native RBD bindings to create the + * RBD image until libvirt creates RBD format 2 by default + */ if (pool.getType() == StoragePoolType.RBD) { format = PhysicalDiskFormat.RAW; + + try { + s_logger.info("Creating RBD image " + pool.getSourcePort() + "/" + name + " with size " + size); + + Rados r = new Rados(pool.getAuthUserName()); + r.confSet("mon_host", pool.getSourceHost() + ":" + pool.getSourcePort()); + r.confSet("key", pool.getAuthSecret()); + r.connect(); + s_logger.debug("Succesfully connected to Ceph cluster at " + r.confGet("mon_host")); + + IoCTX io = r.ioCtxCreate(pool.getSourceDir()); + Rbd rbd = new Rbd(io); + rbd.create(name, size, this.rbdFeatures, this.rbdOrder); + + r.ioCtxDestroy(io); + } catch (RadosException e) { + throw new CloudRuntimeException(e.toString()); + } catch (RbdException e) { + throw new CloudRuntimeException(e.toString()); + } + + volPath = name; + volName = name; + volCapacity = size; + volAllocation = size; + } else { + + if (format == PhysicalDiskFormat.QCOW2) { + libvirtformat = LibvirtStorageVolumeDef.volFormat.QCOW2; + } else if (format == PhysicalDiskFormat.RAW) { + libvirtformat = LibvirtStorageVolumeDef.volFormat.RAW; + } else if (format == PhysicalDiskFormat.DIR) { + libvirtformat = LibvirtStorageVolumeDef.volFormat.DIR; + } else if (format == PhysicalDiskFormat.TAR) { + libvirtformat = LibvirtStorageVolumeDef.volFormat.TAR; + } + + LibvirtStorageVolumeDef volDef = new LibvirtStorageVolumeDef(name, + size, libvirtformat, null, null); + s_logger.debug(volDef.toString()); + try { + StorageVol vol = virtPool.storageVolCreateXML(volDef.toString(), 0); + volPath = vol.getPath(); + volName = vol.getName(); + volAllocation = vol.getInfo().allocation; + volCapacity = vol.getInfo().capacity; + } catch (LibvirtException e) { + throw new CloudRuntimeException(e.toString()); + } } - if (format == PhysicalDiskFormat.QCOW2) { - libvirtformat = LibvirtStorageVolumeDef.volFormat.QCOW2; - } else if (format == PhysicalDiskFormat.RAW) { - libvirtformat = LibvirtStorageVolumeDef.volFormat.RAW; - } else if (format == PhysicalDiskFormat.DIR) { - libvirtformat = LibvirtStorageVolumeDef.volFormat.DIR; - } else if (format == PhysicalDiskFormat.TAR) { - libvirtformat = LibvirtStorageVolumeDef.volFormat.TAR; - } - - LibvirtStorageVolumeDef volDef = new LibvirtStorageVolumeDef(name, - size, libvirtformat, null, null); - s_logger.debug(volDef.toString()); - try { - StorageVol vol = virtPool.storageVolCreateXML(volDef.toString(), 0); - KVMPhysicalDisk disk = new KVMPhysicalDisk(vol.getPath(), - vol.getName(), pool); - disk.setFormat(format); - disk.setSize(vol.getInfo().allocation); - disk.setVirtualSize(vol.getInfo().capacity); - return disk; - } catch (LibvirtException e) { - throw new CloudRuntimeException(e.toString()); - } + KVMPhysicalDisk disk = new KVMPhysicalDisk(volPath, volName, pool); + disk.setFormat(format); + disk.setSize(volAllocation); + disk.setVirtualSize(volCapacity); + return disk; } @Override public boolean deletePhysicalDisk(String uuid, KVMStoragePool pool) { + + /** + * RBD volume can have snapshots and while they exist libvirt + * can't remove the RBD volume + * + * We have to remove those snapshots first + */ + if (pool.getType() == StoragePoolType.RBD) { + try { + s_logger.info("Unprotecting and Removing RBD snapshots of image " + + pool.getSourcePort() + "/" + uuid + " prior to removing the image"); + + Rados r = new Rados(pool.getAuthUserName()); + r.confSet("mon_host", pool.getSourceHost() + ":" + pool.getSourcePort()); + r.confSet("key", pool.getAuthSecret()); + r.connect(); + s_logger.debug("Succesfully connected to Ceph cluster at " + r.confGet("mon_host")); + + IoCTX io = r.ioCtxCreate(pool.getSourceDir()); + Rbd rbd = new Rbd(io); + RbdImage image = rbd.open(uuid); + List snaps = image.snapList(); + for (RbdSnapInfo snap : snaps) { + image.snapUnprotect(snap.name); + image.snapRemove(snap.name); + } + + rbd.close(image); + r.ioCtxDestroy(io); + } catch (RadosException e) { + throw new CloudRuntimeException(e.toString()); + } catch (RbdException e) { + throw new CloudRuntimeException(e.toString()); + } + } + LibvirtStoragePool libvirtPool = (LibvirtStoragePool) pool; try { StorageVol vol = this.getVolume(libvirtPool.getPool(), uuid); @@ -730,11 +811,6 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { * we want to copy it */ - /* Feature 1<<0 means layering in RBD format 2 */ - int rbdFeatures = (1<<0); - /* Order 0 means 4MB blocks (the default) */ - int rbdOrder = 0; - try { if ((srcPool.getSourceHost().equals(destPool.getSourceHost())) && (srcPool.getSourceDir().equals(destPool.getSourceDir()))) { /* We are on the same Ceph cluster, but we require RBD format 2 on the source image */ @@ -755,7 +831,7 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { s_logger.debug("The source image " + srcPool.getSourceDir() + "/" + template.getName() + " is RBD format 1. We have to perform a regular copy (" + template.getVirtualSize() + " bytes)"); - rbd.create(disk.getName(), template.getVirtualSize(), rbdFeatures, rbdOrder); + rbd.create(disk.getName(), template.getVirtualSize(), this.rbdFeatures, this.rbdOrder); RbdImage destImage = rbd.open(disk.getName()); s_logger.debug("Starting to copy " + srcImage.getName() + " to " + destImage.getName() + " in Ceph pool " + srcPool.getSourceDir()); @@ -768,7 +844,7 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { + " is RBD format 2. We will perform a RBD clone using snapshot " + this.rbdTemplateSnapName); /* The source image is format 2, we can do a RBD snapshot+clone (layering) */ - rbd.clone(template.getName(), this.rbdTemplateSnapName, io, disk.getName(), rbdFeatures, rbdOrder); + rbd.clone(template.getName(), this.rbdTemplateSnapName, io, disk.getName(), this.rbdFeatures, this.rbdOrder); s_logger.debug("Succesfully cloned " + template.getName() + "@" + this.rbdTemplateSnapName + " to " + disk.getName()); } @@ -798,7 +874,7 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { s_logger.debug("Creating " + disk.getName() + " on the destination cluster " + rDest.confGet("mon_host") + " in pool " + destPool.getSourceDir()); - dRbd.create(disk.getName(), template.getVirtualSize(), rbdFeatures, rbdOrder); + dRbd.create(disk.getName(), template.getVirtualSize(), this.rbdFeatures, this.rbdOrder); RbdImage srcImage = sRbd.open(template.getName()); RbdImage destImage = dRbd.open(disk.getName()); @@ -943,8 +1019,6 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { */ s_logger.debug("The source image is not RBD, but the destination is. We will convert into RBD format 2"); String tmpFile = "/tmp/" + name; - int rbdFeatures = (1<<0); - int rbdOrder = 0; try { srcFile = new QemuImgFile(sourcePath, sourceFormat); @@ -963,7 +1037,7 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { Rbd rbd = new Rbd(io); s_logger.debug("Creating RBD image " + name + " in Ceph pool " + destPool.getSourceDir() + " with RBD format 2"); - rbd.create(name, disk.getVirtualSize(), rbdFeatures, rbdOrder); + rbd.create(name, disk.getVirtualSize(), this.rbdFeatures, this.rbdOrder); RbdImage image = rbd.open(name); diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/LibvirtStoragePool.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/LibvirtStoragePool.java index 2ce517504d6..bed7b1f652d 100644 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/LibvirtStoragePool.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/LibvirtStoragePool.java @@ -140,7 +140,7 @@ public class LibvirtStoragePool implements KVMStoragePool { @Override public boolean isExternalSnapshot() { - if (this.type == StoragePoolType.Filesystem) { + if (this.type == StoragePoolType.Filesystem || this.type == StoragePoolType.RBD) { return false; } diff --git a/pom.xml b/pom.xml index ff1d3c03c4c..259e81059a8 100644 --- a/pom.xml +++ b/pom.xml @@ -83,9 +83,10 @@ 0.10 build/replace.properties 0.4.9 - 0.1.1 + 0.1.2 target 1.0.10 + 3.0.9 From 478eae8287cb6b34e38eff06199153424f4e2a15 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Thu, 1 Aug 2013 10:04:18 +0200 Subject: [PATCH 040/763] CLOUDSTACK-3947: Override getSyncObjType and getSyncObjId in CreateLBStickinessPolicyCmd.java to prevent concurrent operations on Loadbalancer in a network (cherry picked from commit d571b49d0004a290dc68697680959397c848c0b8) --- .../loadbalancer/CreateLBStickinessPolicyCmd.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/api/src/org/apache/cloudstack/api/command/user/loadbalancer/CreateLBStickinessPolicyCmd.java b/api/src/org/apache/cloudstack/api/command/user/loadbalancer/CreateLBStickinessPolicyCmd.java index 02b253a7c0c..7a2283e4762 100644 --- a/api/src/org/apache/cloudstack/api/command/user/loadbalancer/CreateLBStickinessPolicyCmd.java +++ b/api/src/org/apache/cloudstack/api/command/user/loadbalancer/CreateLBStickinessPolicyCmd.java @@ -22,6 +22,7 @@ import java.util.Map; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseAsyncCmd; import org.apache.cloudstack.api.BaseAsyncCreateCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ServerApiException; @@ -30,6 +31,7 @@ import org.apache.cloudstack.api.response.LBStickinessResponse; import org.apache.log4j.Logger; import com.cloud.event.EventTypes; +import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.NetworkRuleConflictException; import com.cloud.exception.ResourceAllocationException; import com.cloud.exception.ResourceUnavailableException; @@ -156,5 +158,18 @@ public class CreateLBStickinessPolicyCmd extends BaseAsyncCreateCmd { return "creating a Load Balancer Stickiness policy: " + getLBStickinessPolicyName(); } + @Override + public String getSyncObjType() { + return BaseAsyncCmd.networkSyncObject; + } + + @Override + public Long getSyncObjId() { + LoadBalancer lb = _lbService.findById(getLbRuleId()); + if (lb == null) { + throw new InvalidParameterValueException("Unable to find load balancer rule " + getLbRuleId() + " to create stickiness rule"); + } + return lb.getNetworkId(); + } } From b4fe0252230e74d36f62679cd2a751f278edacc1 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Thu, 4 Jul 2013 11:18:35 +0530 Subject: [PATCH 041/763] CLOUDSTACK-3351: Fix add cluster API to set right hypervisor type Signed-off-by: Abhinandan Prateek --- server/src/com/cloud/resource/ResourceManagerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/com/cloud/resource/ResourceManagerImpl.java b/server/src/com/cloud/resource/ResourceManagerImpl.java index 878bfcda414..80037343a5d 100755 --- a/server/src/com/cloud/resource/ResourceManagerImpl.java +++ b/server/src/com/cloud/resource/ResourceManagerImpl.java @@ -455,7 +455,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, List result = new ArrayList(); ClusterVO cluster = new ClusterVO(dcId, podId, clusterName); - cluster.setHypervisorType(cmd.getHypervisor()); + cluster.setHypervisorType(hypervisorType.toString()); cluster.setClusterType(clusterType); cluster.setAllocationState(allocationState); From 4c5a302219ce440ebdd074b8950568821d54504a Mon Sep 17 00:00:00 2001 From: John Burwell Date: Thu, 1 Aug 2013 04:27:30 -0400 Subject: [PATCH 042/763] - CLOUDSTACK-3229: Adds a guard condition to s3xen to prevent double "/" in the resource path if the key starts with a "/" and corrects a log message from adding an additional "/" --- scripts/vm/hypervisor/xenserver/s3xen | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/scripts/vm/hypervisor/xenserver/s3xen b/scripts/vm/hypervisor/xenserver/s3xen index 454097b4458..2798e2b05d2 100644 --- a/scripts/vm/hypervisor/xenserver/s3xen +++ b/scripts/vm/hypervisor/xenserver/s3xen @@ -184,7 +184,12 @@ class S3Client(object): max_error_retry, self.DEFAULT_MAX_ERROR_RETRY) def build_canocialized_resource(self, bucket, key): - return "/" + join([bucket, key], "/") + if not key.startswith("/"): + uri = bucket + "/" + key + else: + uri = bucket + key + + return "/" + uri def noop_send_body(connection): pass @@ -205,9 +210,6 @@ class S3Client(object): headers['Date'] = request_date def perform_request(): - print "method=", method, ", uri=", uri, ", headers=", headers, - " endpoint=", self.end_point - connection = None if self.https_flag: connection = HTTPSConnection(self.end_point) @@ -225,7 +227,7 @@ class S3Client(object): fn_send_body(connection) response = connection.getresponse() - log("Sent " + method + " request to " + self.end_point + "/" + + log("Sent " + method + " request to " + self.end_point + uri + " with headers " + str(headers) + ". Received response status " + str(response.status) + ": " + response.reason) From f744614769afbd7adcb061e6a7a64e3955854ca5 Mon Sep 17 00:00:00 2001 From: Likitha Shetty Date: Thu, 1 Aug 2013 14:10:17 +0530 Subject: [PATCH 043/763] CLOUDSTACK-3981. ListAccounts always sets 'networkTotal' to 0. Add vpc resource count to the response. And set the right values for network resource count. --- .../api/response/AccountResponse.java | 20 +++++++++++++++-- .../api/response/ProjectResponse.java | 22 ++++++++++++++++--- .../ResourceLimitAndCountResponse.java | 6 +++++ .../api/query/dao/AccountJoinDaoImpl.java | 6 ++--- 4 files changed, 46 insertions(+), 8 deletions(-) diff --git a/api/src/org/apache/cloudstack/api/response/AccountResponse.java b/api/src/org/apache/cloudstack/api/response/AccountResponse.java index 0d4f2391f57..957936bb9e3 100644 --- a/api/src/org/apache/cloudstack/api/response/AccountResponse.java +++ b/api/src/org/apache/cloudstack/api/response/AccountResponse.java @@ -19,13 +19,14 @@ package org.apache.cloudstack.api.response; import java.util.List; import java.util.Map; +import com.google.gson.annotations.SerializedName; + import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseResponse; import org.apache.cloudstack.api.EntityReference; import com.cloud.serializer.Param; import com.cloud.user.Account; -import com.google.gson.annotations.SerializedName; @SuppressWarnings("unused") @EntityReference(value = Account.class) @@ -189,7 +190,7 @@ public class AccountResponse extends BaseResponse implements ResourceLimitAndCou @Override public String getObjectId() { - return this.id; + return id; } public void setId(String id) { @@ -351,7 +352,22 @@ public class AccountResponse extends BaseResponse implements ResourceLimitAndCou public void setNetworkAvailable(String networkAvailable) { this.networkAvailable = networkAvailable; } + + @Override + public void setVpcLimit(String vpcLimit) { + this.vpcLimit = networkLimit; + } + + @Override + public void setVpcTotal(Long vpcTotal) { + this.vpcTotal = vpcTotal; + } + @Override + public void setVpcAvailable(String vpcAvailable) { + this.vpcAvailable = vpcAvailable; + } + @Override public void setCpuLimit(String cpuLimit) { this.cpuLimit = cpuLimit; diff --git a/api/src/org/apache/cloudstack/api/response/ProjectResponse.java b/api/src/org/apache/cloudstack/api/response/ProjectResponse.java index 4fdd1374495..81b51c8b34d 100644 --- a/api/src/org/apache/cloudstack/api/response/ProjectResponse.java +++ b/api/src/org/apache/cloudstack/api/response/ProjectResponse.java @@ -19,13 +19,14 @@ package org.apache.cloudstack.api.response; import java.util.ArrayList; import java.util.List; +import com.google.gson.annotations.SerializedName; + import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseResponse; import org.apache.cloudstack.api.EntityReference; import com.cloud.projects.Project; import com.cloud.serializer.Param; -import com.google.gson.annotations.SerializedName; @EntityReference(value=Project.class) @SuppressWarnings("unused") @@ -182,7 +183,7 @@ public class ProjectResponse extends BaseResponse implements ResourceLimitAndCou } public void setOwner(String owner) { - this.ownerName = owner; + ownerName = owner; } public void setState(String state) { @@ -194,7 +195,7 @@ public class ProjectResponse extends BaseResponse implements ResourceLimitAndCou } public void addTag(ResourceTagResponse tag){ - this.tags.add(tag); + tags.add(tag); } @Override @@ -296,7 +297,22 @@ public class ProjectResponse extends BaseResponse implements ResourceLimitAndCou public void setNetworkAvailable(String networkAvailable) { this.networkAvailable = networkAvailable; } + + @Override + public void setVpcLimit(String vpcLimit) { + this.vpcLimit = networkLimit; + } + + @Override + public void setVpcTotal(Long vpcTotal) { + this.vpcTotal = vpcTotal; + } + @Override + public void setVpcAvailable(String vpcAvailable) { + this.vpcAvailable = vpcAvailable; + } + @Override public void setCpuLimit(String cpuLimit) { this.cpuLimit = cpuLimit; diff --git a/api/src/org/apache/cloudstack/api/response/ResourceLimitAndCountResponse.java b/api/src/org/apache/cloudstack/api/response/ResourceLimitAndCountResponse.java index 57aabdd2c08..49bb4e02c07 100644 --- a/api/src/org/apache/cloudstack/api/response/ResourceLimitAndCountResponse.java +++ b/api/src/org/apache/cloudstack/api/response/ResourceLimitAndCountResponse.java @@ -29,6 +29,12 @@ public interface ResourceLimitAndCountResponse { public void setNetworkAvailable(String networkAvailable); + public void setVpcLimit(String vpcLimit); + + public void setVpcTotal(Long vpcTotal); + + public void setVpcAvailable(String vpcAvailable); + public void setCpuLimit(String cpuLimit); public void setCpuTotal(Long cpuTotal); diff --git a/server/src/com/cloud/api/query/dao/AccountJoinDaoImpl.java b/server/src/com/cloud/api/query/dao/AccountJoinDaoImpl.java index d83e60a5f11..edbd3ae2537 100644 --- a/server/src/com/cloud/api/query/dao/AccountJoinDaoImpl.java +++ b/server/src/com/cloud/api/query/dao/AccountJoinDaoImpl.java @@ -173,9 +173,9 @@ public class AccountJoinDaoImpl extends GenericDaoBase impl String vpcLimitDisplay = (accountIsAdmin || vpcLimit == -1) ? "Unlimited" : String.valueOf(vpcLimit); long vpcTotal = (account.getVpcTotal() == null) ? 0 : account.getVpcTotal(); String vpcAvail = (accountIsAdmin || vpcLimit == -1) ? "Unlimited" : String.valueOf(vpcLimit - vpcTotal); - response.setNetworkLimit(vpcLimitDisplay); - response.setNetworkTotal(vpcTotal); - response.setNetworkAvailable(vpcAvail); + response.setVpcLimit(vpcLimitDisplay); + response.setVpcTotal(vpcTotal); + response.setVpcAvailable(vpcAvail); //get resource limits for cpu cores long cpuLimit = ApiDBUtils.findCorrectResourceLimit(account.getCpuLimit(), account.getType(), ResourceType.cpu); From f7555c1ec10de0d4ac297a04440d8c4d98b540a7 Mon Sep 17 00:00:00 2001 From: Radhika PC Date: Thu, 1 Aug 2013 14:41:07 +0530 Subject: [PATCH 044/763] CLOUDSTACK-3236 --- docs/en-US/portable-ip.xml | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/docs/en-US/portable-ip.xml b/docs/en-US/portable-ip.xml index 83d5b43b206..68b759b65de 100644 --- a/docs/en-US/portable-ip.xml +++ b/docs/en-US/portable-ip.xml @@ -24,10 +24,10 @@ About Portable IP Portable IPs in &PRODUCT; are region-level pool of IPs, which are elastic in nature, that can be transferred across geographically separated zones. As an administrator, you can - provision a pool of portable IPs at region level and are available for user consumption. The - users can acquire portable IPs if admin has provisioned portable public IPs at the region - level they are part of. These IPs can be use for any service within an advanced zone. You can - also use portable IPs for EIP services in basic zones. + provision a pool of portable public IPs at region level and are available for user + consumption. The users can acquire portable IPs if admin has provisioned portable IPs at the + region level they are part of. These IPs can be use for any service within an advanced zone. + You can also use portable IPs for EIP services in basic zones. The salient features of Portable IP are as follows: IP is statically allocated @@ -42,19 +42,17 @@ IP is transferable across both Basic and Advanced zones - IP is transferable across VPC, non-VPC Isolated and Shared networks - - - - - - + IP is transferable across VPC, non-VPC isolated and shared networks + + Guidelines + Before transferring to another network, ensure that no network rules (Firewall, Static + NAT, Port Forwarding, and so on) exist on that portable IP. +
Configuring Portable IPs - Log in to the &PRODUCT; UI as an administrator or end user. @@ -129,4 +127,16 @@
+
+ Transferring Portable IP + An IP can be transferred from one network to another only if Static NAT is enabled. + However, when a portable IP is associated with a network, you can use it for any service in + the network. + To transfer a portable IP across the networks, execute the following API: + http://localhost:8096/client/api?command=enableStaticNat&response=json&ipaddressid=a4bc37b2-4b4e-461d-9a62-b66414618e36&virtualmachineid=a242c476-ef37-441e-9c7b-b303e2a9cb4f&networkid=6e7cd8d1-d1ba-4c35-bdaf-333354cbd49810 + Replace the UUID with appropriate UUID. For example, if you want to transfer a portable IP + to network X and VM Y in a network, execute the following: + http://localhost:8096/client/api?command=enableStaticNat&response=json&ipaddressid=a4bc37b2-4b4e-461d-9a62-b66414618e36&virtualmachineid=Y&networkid=X + +
From 57a82a1715d88e8fb74fe0b7c2997703d33b7b49 Mon Sep 17 00:00:00 2001 From: Radhika PC Date: Thu, 1 Aug 2013 15:51:23 +0530 Subject: [PATCH 045/763] https://issues.apache.org/jira/browse/CLOUDSTACK-817 --- docs/en-US/images/addAccount-icon.png | Bin 0 -> 605 bytes docs/en-US/ip-vlan-tenant.xml | 2 +- docs/en-US/networks.xml | 9 +++++---- 3 files changed, 6 insertions(+), 5 deletions(-) create mode 100644 docs/en-US/images/addAccount-icon.png diff --git a/docs/en-US/images/addAccount-icon.png b/docs/en-US/images/addAccount-icon.png new file mode 100644 index 0000000000000000000000000000000000000000..4743dbef2cfe89733daf60ede4099f2fff4cdd5d GIT binary patch literal 605 zcmV-j0;2tiP)Px#1ZP1_K>z@;j|==^1poj532;bRa{vGi!vFvd!vV){sAK>D0rE*iK~y+TozmG) z8c`GnVE;ygQmmDV!T`fIG7htE3`GNKNi}_`EhV%F4j5YvT20^DCP-7o7zA6lq_Qmo0;$2<#)QyFPa_9R;$(30Ry5fH@6Dop|A~gPCbtt93E?y){wc4 z{J>L_;qicRa9D5XCkVP)tN)KRBrnKj#Eug*N%JE(`NU|-fB$j6tW;}Wf+9$k7ZqL# zaDw0I_GVW<-7n!_yeQM01nhQVxG%f%0134N!4UITa-uRBnVx+e-7FMp^~RlTHSW2* zoTV6n75sBab*H%dHWtT5a9)rSbE#5!@1NI@^~RqUGcV)slBIGv5FCTove+C(lUdhm zaJoFjV(AuZ=lT2OB$v-KoM^G4aL@F@Vl19;W21vZ7Q#nr%WAEWnpfXMV=Jp`z<5a! z_qF;~{*!-^CKRi;^j0LfG>-3{OYVpI8%Y)CSGtK5D{Cd&6yt=x6H5)~2 zj<)>yYd?`pnh|S{zHb<@Lx5wU2}r`}@rEZODK&F?c76bLGYi4-Fnn)Iieh>m^_i_G zbPUak6e~(faBclFv;^wHbb7;=uisK?8afe*3bN9ch!ypC2`~^K#K`h8EaUGJAD6Nu rBLH@|F*pe@|Fcjox6zjHzMYL5w0H9d-#d%y00000NkvXXu0mjfVYLi` literal 0 HcmV?d00001 diff --git a/docs/en-US/ip-vlan-tenant.xml b/docs/en-US/ip-vlan-tenant.xml index 42124f0f446..f66fde1661f 100644 --- a/docs/en-US/ip-vlan-tenant.xml +++ b/docs/en-US/ip-vlan-tenant.xml @@ -19,7 +19,7 @@ under the License. -->
- Dedicated Resources: Public IP Addresses and VLANs Per Account + Reserving Public IP Addresses and VLANs Per Account &PRODUCT; provides you the ability to reserve a set of public IP addresses and VLANs exclusively for an account. During zone creation, you can continue to define a set of VLANs and multiple public IP ranges. This feature extends the functionality to enable you to dedicate a diff --git a/docs/en-US/networks.xml b/docs/en-US/networks.xml index 5d0c82ab8d6..b28f985a147 100644 --- a/docs/en-US/networks.xml +++ b/docs/en-US/networks.xml @@ -32,8 +32,13 @@ xmlns:xi="http://www.w3.org/2001/XInclude"/> + + + + + @@ -47,10 +52,6 @@ - - - - From 01449d48f97020422cc1671af860a0bed8a048af Mon Sep 17 00:00:00 2001 From: Prasanna Santhanam Date: Thu, 1 Aug 2013 16:09:07 +0530 Subject: [PATCH 046/763] CLOUDSTACK-3999: Fix the skipTest typo Signed-off-by: Prasanna Santhanam (cherry picked from commit cf6ce42b536a287d36a09b776ae1b756cb1c14ee) --- .../test_redundant_router_deployment_planning.py | 8 ++++---- test/integration/smoke/test_deploy_vm_with_userdata.py | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/test/integration/component/test_redundant_router_deployment_planning.py b/test/integration/component/test_redundant_router_deployment_planning.py index c8aac06c033..75fc3abed96 100644 --- a/test/integration/component/test_redundant_router_deployment_planning.py +++ b/test/integration/component/test_redundant_router_deployment_planning.py @@ -233,7 +233,7 @@ class TestRvRDeploymentPlanning(cloudstackTestCase): ) if len(pods) < 2: - raise self.SkipTest("The env don't have 2 pods req for test") + raise self.skipTest("The env don't have 2 pods req for test") # Creating network using the network offering created self.debug("Creating network with network offering: %s" % @@ -380,7 +380,7 @@ class TestRvRDeploymentPlanning(cloudstackTestCase): "List clusters should not return empty response" ) if len(clusters) < 2: - raise self.SkipTest( + raise self.skipTest( "The env don't have 2 clusters req for test") self.debug("disable all pods except one!") @@ -600,7 +600,7 @@ class TestRvRDeploymentPlanning(cloudstackTestCase): ) if len(storage_pools) < 2: - raise self.SkipTest( + raise self.skipTest( "The env don't have 2 storage pools req for test") self.debug("disable all pods except one!") @@ -841,7 +841,7 @@ class TestRvRDeploymentPlanning(cloudstackTestCase): ) if len(hosts) < 2: - raise self.SkipTest( + raise self.skipTest( "The env don't have 2 hosts req for test") self.debug("disable all pods except one!") diff --git a/test/integration/smoke/test_deploy_vm_with_userdata.py b/test/integration/smoke/test_deploy_vm_with_userdata.py index 260106cbb0f..8910b2e8f89 100644 --- a/test/integration/smoke/test_deploy_vm_with_userdata.py +++ b/test/integration/smoke/test_deploy_vm_with_userdata.py @@ -71,6 +71,7 @@ class TestDeployVmWithUserData(cloudstackTestCase): cls.services["service_offering"] ) cls.account = Account.create(cls.apiClient, services=cls.services["account"]) + cls.cleanup = [cls.account] cls.template = get_template( cls.apiClient, cls.zone.id, @@ -79,7 +80,7 @@ class TestDeployVmWithUserData(cloudstackTestCase): cls.debug("Successfully created account: %s, id: \ %s" % (cls.account.name,\ cls.account.id)) - cls.cleanup = [cls.account] + # Generate userdata of 2500 bytes. This is larger than the 2048 bytes limit. # CS however allows for upto 4K bytes in the code. So this must succeed. From 712d9629430091075f3aee7398465962893826d2 Mon Sep 17 00:00:00 2001 From: Prasanna Santhanam Date: Thu, 1 Aug 2013 16:09:59 +0530 Subject: [PATCH 047/763] Remove unused code Signed-off-by: Prasanna Santhanam (cherry picked from commit 7452bb858703d9a24e6d4bb50036851e06c8418a) --- tools/marvin/marvin/cloudstackTestCase.py | 3 --- tools/marvin/marvin/marvinPlugin.py | 2 -- 2 files changed, 5 deletions(-) diff --git a/tools/marvin/marvin/cloudstackTestCase.py b/tools/marvin/marvin/cloudstackTestCase.py index b04342be58c..85ef5423091 100644 --- a/tools/marvin/marvin/cloudstackTestCase.py +++ b/tools/marvin/marvin/cloudstackTestCase.py @@ -35,9 +35,6 @@ def user(Name, DomainName, AcctType): class cloudstackTestCase(unittest.case.TestCase): clstestclient = None - def __init__(self, args): - unittest.case.TestCase.__init__(self, args) - @classmethod def getClsTestClient(cls): return cls.clstestclient diff --git a/tools/marvin/marvin/marvinPlugin.py b/tools/marvin/marvin/marvinPlugin.py index f7b0664856c..1c6eb849ef0 100644 --- a/tools/marvin/marvin/marvinPlugin.py +++ b/tools/marvin/marvin/marvinPlugin.py @@ -22,8 +22,6 @@ import nose.core from marvin.cloudstackTestCase import cloudstackTestCase from marvin import deployDataCenter from nose.plugins.base import Plugin -from functools import partial - class MarvinPlugin(Plugin): """ From 582e3a37a44c0db15d6a45a4ff1d12c3f7762621 Mon Sep 17 00:00:00 2001 From: Prasanna Santhanam Date: Thu, 1 Aug 2013 16:23:16 +0530 Subject: [PATCH 048/763] Include the exception from urllib in wget test Signed-off-by: Prasanna Santhanam (cherry picked from commit eb5099a2a25ac509e7455d0652d90b361ec3971d) --- test/integration/component/test_vpc_network_lbrules.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/integration/component/test_vpc_network_lbrules.py b/test/integration/component/test_vpc_network_lbrules.py index 5aa626ab8a8..de29ce19a97 100644 --- a/test/integration/component/test_vpc_network_lbrules.py +++ b/test/integration/component/test_vpc_network_lbrules.py @@ -339,14 +339,14 @@ class TestVPCNetworkLBRules(cloudstackTestCase): try: urllib.urlretrieve("http://%s/test.html" % public_ip.ipaddress.ipaddress, filename="test.html") if not testnegative: - self.debug("Successesfull to wget from VM=%s http server on public_ip=%s" % (vm.name, public_ip.ipaddress.ipaddress)) + self.debug("Successful to wget from VM=%s http server on public_ip=%s" % (vm.name, public_ip.ipaddress.ipaddress)) else: - self.fail("Successesfull to wget from VM=%s http server on public_ip=%s" % (vm.name, public_ip.ipaddress.ipaddress)) - except: + self.fail("Successful to wget from VM=%s http server on public_ip=%s" % (vm.name, public_ip.ipaddress.ipaddress)) + except Exception, e: if not testnegative: - self.fail("Failed to wget from VM=%s http server on public_ip=%s" % (vm.name, public_ip.ipaddress.ipaddress)) + self.fail("Failed to wget from VM=%s http server on public_ip=%s because of %s" % (vm.name, public_ip.ipaddress.ipaddress, e)) else: - self.debug("Failed to wget from VM=%s http server on public_ip=%s" % (vm.name, public_ip.ipaddress.ipaddress)) + self.debug("Failed to wget from VM=%s http server on public_ip=%s because of %s" % (vm.name, public_ip.ipaddress.ipaddress, e)) def create_StaticNatRule_For_VM(self, vm, public_ip, network): self.debug("Enabling static NAT for IP: %s" % From f064ad22dc55e0e5e7b5165f1864f2339d39b8f4 Mon Sep 17 00:00:00 2001 From: Prasanna Santhanam Date: Thu, 1 Aug 2013 16:42:28 +0530 Subject: [PATCH 049/763] CLOUDSTACK-3989: We don't hold the vm object VirtualMachine.list( only gives the list of vms but not the vm object as returned by VirtualMachine.create( any operations on the listed VMs then fails. Explicit call to startVirtualMachine as in setUpClass fixes this. Signed-off-by: Prasanna Santhanam --- test/integration/component/test_vpc_network.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/integration/component/test_vpc_network.py b/test/integration/component/test_vpc_network.py index 03ff71e877a..4e6c02c1045 100644 --- a/test/integration/component/test_vpc_network.py +++ b/test/integration/component/test_vpc_network.py @@ -2283,7 +2283,9 @@ class TestVPCNetworkGc(cloudstackTestCase): ) for vm in vms: if vm.state == "Stopped": - vm.start(self.apiclient) + cmd = startVirtualMachine.startVirtualMachineCmd() + cmd.id = vm.id + self.apiclient.startVirtualMachine(cmd) return def validate_vpc_offering(self, vpc_offering): From 087b1dcfc303b51621380801b86a95d2efbbe4b6 Mon Sep 17 00:00:00 2001 From: Bharat Kumar Date: Mon, 1 Jul 2013 18:08:00 +0530 Subject: [PATCH 050/763] Cloudstack-3299 incorrect arguments sent to the script createIpalias in case of vmware, Cloudstack-3240 corrected a mistake in the script name. Signed-off-by: Abhinandan Prateek --- .../vmware/resource/VmwareResource.java | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java index a33b94e9ff0..1216e17a7a3 100755 --- a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -2132,36 +2132,36 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa protected Answer execute(final CreateIpAliasCommand cmd) { if (s_logger.isInfoEnabled()) { - s_logger.info("Executing createipAlias command: " + _gson.toJson(cmd)); + s_logger.info("Executing createIpAlias command: " + _gson.toJson(cmd)); } String routerIp = cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP); List ipAliasTOs = cmd.getIpAliasList(); - String args=routerIp+" "; + String args=""; for (IpAliasTO ipaliasto : ipAliasTOs) { args = args + ipaliasto.getAlias_count()+":"+ipaliasto.getRouterip()+":"+ipaliasto.getNetmask()+"-"; } if (s_logger.isDebugEnabled()) { - s_logger.debug("Run command on domR " + cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP) + ", /root/createipAlias " + args); + s_logger.debug("Run command on domR " + cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP) + ", /root/createIpAlias " + args); } try { VmwareManager mgr = getServiceContext().getStockObject(VmwareManager.CONTEXT_STOCK_NAME); String controlIp = getRouterSshControlIp(cmd); Pair result = SshHelper.sshExecute(controlIp, DEFAULT_DOMR_SSHPORT, "root", mgr.getSystemVMKeyFile(), null, - "/root/createipAlias.sh " + args); + "/root/createIpAlias.sh " + args); if (!result.first()) { - s_logger.error("ipAlias command on domr " + controlIp + " failed, message: " + result.second()); + s_logger.error("CreateIpAlias command on domr " + controlIp + " failed, message: " + result.second()); return new Answer(cmd, false, "createipAlias failed due to " + result.second()); } if (s_logger.isInfoEnabled()) { - s_logger.info("createipAlias command on domain router " + controlIp + " completed"); + s_logger.info("createIpAlias command on domain router " + controlIp + " completed"); } } catch (Throwable e) { - String msg = "createipAlias failed due to " + VmwareHelper.getExceptionMessage(e); + String msg = "createIpAlias failed due to " + VmwareHelper.getExceptionMessage(e); s_logger.error(msg, e); return new Answer(cmd, false, msg); } @@ -2174,9 +2174,9 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa List revokedIpAliasTOs = cmd.getDeleteIpAliasTos(); List activeIpAliasTOs = cmd.getCreateIpAliasTos(); if (s_logger.isInfoEnabled()) { - s_logger.info("Executing deleteipAlias command: " + _gson.toJson(cmd)); + s_logger.info("Executing deleteIpAlias command: " + _gson.toJson(cmd)); } - String args=routerIp+" "; + String args=""; for (IpAliasTO ipAliasTO : revokedIpAliasTOs) { args = args + ipAliasTO.getAlias_count()+":"+ipAliasTO.getRouterip()+":"+ipAliasTO.getNetmask()+"-"; } @@ -2185,27 +2185,27 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa args = args + ipAliasTO.getAlias_count()+":"+ipAliasTO.getRouterip()+":"+ipAliasTO.getNetmask()+"-"; } if (s_logger.isDebugEnabled()) { - s_logger.debug("Run command on domR " + cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP) + ", /root/deleteipAlias " + args); + s_logger.debug("Run command on domR " + cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP) + ", /root/deleteIpAlias " + args); } try { VmwareManager mgr = getServiceContext().getStockObject(VmwareManager.CONTEXT_STOCK_NAME); String controlIp = getRouterSshControlIp(cmd); Pair result = SshHelper.sshExecute(controlIp, DEFAULT_DOMR_SSHPORT, "root", mgr.getSystemVMKeyFile(), null, - "/root/deleteipAlias.sh " + args); + "/root/deleteIpAlias.sh " + args); if (!result.first()) { - s_logger.error("ipAlias command on domr " + controlIp + " failed, message: " + result.second()); + s_logger.error("deleteIpAlias command on domr " + controlIp + " failed, message: " + result.second()); - return new Answer(cmd, false, "deleteipAlias failed due to " + result.second()); + return new Answer(cmd, false, "deleteIpAlias failed due to " + result.second()); } if (s_logger.isInfoEnabled()) { - s_logger.info("deleteipAlias command on domain router " + controlIp + " completed"); + s_logger.info("deleteIpAlias command on domain router " + controlIp + " completed"); } } catch (Throwable e) { - String msg = "deleteipAlias failed due to " + VmwareHelper.getExceptionMessage(e); + String msg = "deleteIpAlias failed due to " + VmwareHelper.getExceptionMessage(e); s_logger.error(msg, e); return new Answer(cmd, false, msg); } From 22a3ba18c380f3c2ab68304f0ec6ccb17b287710 Mon Sep 17 00:00:00 2001 From: Nitin Mehta Date: Thu, 1 Aug 2013 17:36:40 +0530 Subject: [PATCH 051/763] =CLOUDSTACK-3102 ResourceDetail API - If you add detail for any resource other than volume and nic then throw an exception --- .../src/com/cloud/metadata/ResourceMetaDataManagerImpl.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/src/com/cloud/metadata/ResourceMetaDataManagerImpl.java b/server/src/com/cloud/metadata/ResourceMetaDataManagerImpl.java index 23708f862e1..f4b76394be5 100644 --- a/server/src/com/cloud/metadata/ResourceMetaDataManagerImpl.java +++ b/server/src/com/cloud/metadata/ResourceMetaDataManagerImpl.java @@ -214,9 +214,11 @@ public class ResourceMetaDataManagerImpl extends ManagerBase implements Resource if(resourceType == TaggedResourceType.Volume){ VolumeDetailVO v = new VolumeDetailVO(id, key, value); _volumeDetailDao.persist(v); - }else { + }else if (resourceType == TaggedResourceType.Nic){ NicDetailVO n = new NicDetailVO(id, key, value); _nicDetailDao.persist(n); + }else{ + throw new InvalidParameterValueException("The resource type " + resourceType + " is not supported by the API yet"); } } From 3388291cc571c5bb5835c411f04f08bbb1516024 Mon Sep 17 00:00:00 2001 From: Prasanna Santhanam Date: Thu, 1 Aug 2013 19:24:50 +0530 Subject: [PATCH 052/763] CLOUDSTACK-3921: Don't change order of params in PublicIpAddress.create() Or existing tests that depend on the order barf and fail. Signed-off-by: Prasanna Santhanam --- tools/marvin/marvin/integration/lib/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/marvin/marvin/integration/lib/base.py b/tools/marvin/marvin/integration/lib/base.py index bbf8440f725..2a2c65b4127 100755 --- a/tools/marvin/marvin/integration/lib/base.py +++ b/tools/marvin/marvin/integration/lib/base.py @@ -1073,8 +1073,8 @@ class PublicIPAddress: self.__dict__.update(items) @classmethod - def create(cls, apiclient, accountid=None, zoneid=None, domainid=None, - isportable=None, services=None, networkid=None, projectid=None, vpcid=None): + def create(cls, apiclient, accountid=None, zoneid=None, domainid=None, services=None, + networkid=None, projectid=None, vpcid=None, isportable=None): """Associate Public IP address""" cmd = associateIpAddress.associateIpAddressCmd() From 1b14fa79532601d4cd7335c622112ca59cf70b35 Mon Sep 17 00:00:00 2001 From: Kishan Kavala Date: Thu, 1 Aug 2013 19:53:13 +0530 Subject: [PATCH 053/763] CLOUDSTACK-3979: Update fullBackup flag correctly based on snapshot delta max config --- .../storage/snapshot/XenserverSnapshotStrategy.java | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/XenserverSnapshotStrategy.java b/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/XenserverSnapshotStrategy.java index 3e6c508a2ea..10f3a4c5fdd 100644 --- a/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/XenserverSnapshotStrategy.java +++ b/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/XenserverSnapshotStrategy.java @@ -102,14 +102,10 @@ public class XenserverSnapshotStrategy extends SnapshotStrategyBase { SnapshotManager.DELTAMAX); int deltaSnap = _deltaSnapshotMax; + SnapshotDataStoreVO parentSnapshotOnBackupStore = this.snapshotStoreDao.findBySnapshot(parentSnapshot.getId(), + DataStoreRole.Image); int i; - SnapshotDataStoreVO parentSnapshotOnBackupStore = null; - for (i = 1; i < deltaSnap; i++) { - parentSnapshotOnBackupStore = this.snapshotStoreDao.findBySnapshot(parentSnapshot.getId(), - DataStoreRole.Image); - if (parentSnapshotOnBackupStore == null) { - break; - } + for (i = 1; (i < deltaSnap && (parentSnapshotOnBackupStore != null)); i++) { Long prevBackupId = parentSnapshotOnBackupStore.getParentSnapshotId(); if (prevBackupId == 0) { From 47803c21fb66acf786a3c6df99596c41f621b4fe Mon Sep 17 00:00:00 2001 From: Sanjay Tripathi Date: Thu, 1 Aug 2013 13:17:57 +0530 Subject: [PATCH 054/763] CLOUDSTACK-4001: Unable to deploy Windows server 2012 64 bit VM on xenserver 6.2 Signed-off-by: Abhinandan Prateek --- .../xen/src/com/cloud/hypervisor/xen/resource/CitrixHelper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixHelper.java b/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixHelper.java index 0d93d839be6..9cdbd8410d2 100644 --- a/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixHelper.java +++ b/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixHelper.java @@ -1431,7 +1431,7 @@ public class CitrixHelper { _xenServer620GuestOsMap.put("Windows Server 2008 SP2 (64-bit)", "Windows Server 2008 SP2 (64-bit)"); _xenServer620GuestOsMap.put("Windows Server 2008 R2 (64-bit)", "Windows Server 2008 R2 (64-bit)"); _xenServer620GuestOsMap.put("Windows Server 2008 R2 SP1 (64-bit)", "Windows Server 2008 R2 SP1 (64-bit)"); - _xenServer620GuestOsMap.put("Windows Server 2012 (64-bit)", "Windows Server 2012 (64-bit) (experimental)"); + _xenServer620GuestOsMap.put("Windows Server 2012 (64-bit)", "Windows Server 2012 (64-bit)"); _xenServer620GuestOsMap.put("Windows Vista SP2 (32-bit)", "Windows Vista (32-bit)"); _xenServer620GuestOsMap.put("Windows XP SP3 (32-bit)", "Windows XP SP3 (32-bit)"); _xenServer620GuestOsMap.put("Ubuntu 10.04 (32-bit)", "Ubuntu Lucid Lynx 10.04 (32-bit)"); From 27082fa516b9e5545410e5364a98dbfcbed02075 Mon Sep 17 00:00:00 2001 From: Prasanna Santhanam Date: Thu, 1 Aug 2013 21:17:10 +0530 Subject: [PATCH 055/763] CLOUDSTACK-3865: Additional debug logging for size mismatch Signed-off-by: Prasanna Santhanam --- test/integration/component/test_blocker_bugs.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/test/integration/component/test_blocker_bugs.py b/test/integration/component/test_blocker_bugs.py index 0e2b2c0db11..2cdc2707020 100644 --- a/test/integration/component/test_blocker_bugs.py +++ b/test/integration/component/test_blocker_bugs.py @@ -25,7 +25,6 @@ from marvin.integration.lib.common import * #Import Local Modules from marvin.cloudstackTestCase import * from marvin.cloudstackAPI import * -from marvin.remoteSSHClient import remoteSSHClient class Services: @@ -720,7 +719,7 @@ class TestTemplates(cloudstackTestCase): cls.services["account"] = cls.account.name cls.service_offering = ServiceOffering.create( cls.api_client, - cls.services["service_offering"] + cls.services["service_offering"], ) # create virtual machine @@ -888,13 +887,13 @@ class TestTemplates(cloudstackTestCase): self.assertEqual( templates[0].size, self.volume.size, - "Check if size of snapshot and template matches" + "Derived template size (%s) does not match snapshot size (%s)" % (templates[0].size, self.volume.size) ) return @attr(speed = "slow") @attr(tags = ["advanced", "advancedns", "basic", "sg", "eip"]) - def test_03_resuse_template_name(self): + def test_03_reuse_template_name(self): """TS_BUG_011-Test Reusing deleted template name """ From 35492a3668f261d4dcf7b79e6cdabcb1f36483c6 Mon Sep 17 00:00:00 2001 From: Prasanna Santhanam Date: Thu, 1 Aug 2013 21:49:39 +0530 Subject: [PATCH 056/763] CLOUDSTACK-2921: Fix the change service offering test on stoppedvm There is no need to attach a volume to the VM when doing the changeserviceOffering test. Condensed the test to verify that the VM's offering is changed successfully when stopped and reflects when VM is started. Signed-off-by: Prasanna Santhanam --- test/integration/component/test_stopped_vm.py | 56 +++---------------- 1 file changed, 8 insertions(+), 48 deletions(-) diff --git a/test/integration/component/test_stopped_vm.py b/test/integration/component/test_stopped_vm.py index 212239cf417..cde2bad7a32 100644 --- a/test/integration/component/test_stopped_vm.py +++ b/test/integration/component/test_stopped_vm.py @@ -432,15 +432,13 @@ class TestDeployVM(cloudstackTestCase): @attr(tags = ["advanced", "eip", "advancedns", "basic", "sg"]) def test_05_deploy_startvm_false_change_so(self): - """Test Deploy Virtual Machine with startVM=false and - change service offering + """Test Deploy Virtual Machine with startVM=false and change service offering """ # Validate the following: # 1. deploy Vm with the startvm=false. Attach volume to the instance # 2. listVM command should return the deployed VM.State of this VM # should be "Stopped". - # 3. Attach volume should be successful # 4. Change service offering self.debug("Deploying instance in the account: %s" % @@ -452,7 +450,6 @@ class TestDeployVM(cloudstackTestCase): domainid=self.account.domainid, serviceofferingid=self.service_offering.id, startvm=False, - diskofferingid=self.disk_offering.id, ) self.debug("Deployed instance in account: %s" % @@ -479,37 +476,11 @@ class TestDeployVM(cloudstackTestCase): "Stopped", "VM should be in Stopped state after deployment with startvm=false" ) - self.debug("Creating a volume in account: %s" % - self.account.name) - volume = Volume.create( + medium_service_off = ServiceOffering.create( self.apiclient, - self.services["volume"], - zoneid=self.zone.id, - account=self.account.name, - domainid=self.account.domainid, - diskofferingid=self.disk_offering.id + self.services["service_offering"] ) - self.debug("Created volume in account: %s" % self.account.name) - self.debug("Attaching volume to instance: %s" % - self.virtual_machine.name) - try: - self.virtual_machine.attach_volume(self.apiclient, volume) - except Exception as e: - self.fail("Attach volume failed!") - self.debug("Fetching details of medium service offering") - medium_service_offs = ServiceOffering.list( - self.apiclient, - name="Medium Instance" - ) - if isinstance(medium_service_offs, list): - medium_service_off = medium_service_offs[0] - else: - self.debug("Service offering not found! Creating a new one..") - medium_service_off = ServiceOffering.create( - self.apiclient, - self.services["service_offering"] - ) - self.cleanup.append(medium_service_off) + self.cleanup.append(medium_service_off) self.debug("Changing service offering for instance: %s" % self.virtual_machine.name) @@ -525,22 +496,11 @@ class TestDeployVM(cloudstackTestCase): self.virtual_machine.start(self.apiclient) self.debug("Instance: %s started" % self.virtual_machine.name) - self.debug("Detaching the disk: %s" % volume.name) - self.virtual_machine.detach_volume(self.apiclient, volume) - self.debug("Datadisk %s detached!" % volume.name) + listedvm = VirtualMachine.list(self.apiclient, id=self.virtual_machine.id) + self.assert_(isinstance(listedvm, list)) + self.assert_(len(listedvm) > 0) + self.assertEqual(listedvm[0].serviceofferingid, medium_service_off.id, msg="VM did not change service offering") - volumes = Volume.list( - self.apiclient, - virtualmachineid=self.virtual_machine.id, - type='DATADISK', - id=volume.id, - listall=True - ) - self.assertEqual( - volumes, - None, - "List Volumes should not list any volume for instance" - ) return @attr(tags = ["advanced", "eip", "advancedns", "basic", "sg"]) From ba059dd0ae09f6f7854aef5d14cccb9ce948724b Mon Sep 17 00:00:00 2001 From: Prasanna Santhanam Date: Thu, 1 Aug 2013 22:23:04 +0530 Subject: [PATCH 057/763] CLOUDSTACK-2921: Added more logging to attach failure Also reduced size of attached disk to 1GB Signed-off-by: Prasanna Santhanam --- test/integration/component/test_stopped_vm.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/test/integration/component/test_stopped_vm.py b/test/integration/component/test_stopped_vm.py index cde2bad7a32..e1f5f2eb2f8 100644 --- a/test/integration/component/test_stopped_vm.py +++ b/test/integration/component/test_stopped_vm.py @@ -65,9 +65,9 @@ class Services: "memory": 128, # In MBs }, "disk_offering": { - "displaytext": "Small volume", - "name": "Small volume", - "disksize": 20 + "displaytext": "Tiny volume", + "name": "Tiny volume", + "disksize": 1 }, "volume": { "diskname": "DataDisk", @@ -680,9 +680,8 @@ class TestDeployVM(cloudstackTestCase): return @attr(tags = ["advanced", "eip", "advancedns", "basic", "sg"]) - def test_08_deploy_attach_volume(self): - """Test Deploy Virtual Machine with startVM=false and - attach volume already attached to different machine + def test_08_deploy_attached_volume(self): + """Test Deploy Virtual Machine with startVM=false and attach volume already attached to different machine """ # Validate the following: @@ -794,7 +793,7 @@ class TestDeployVM(cloudstackTestCase): try: self.virtual_machine_1.attach_volume(self.apiclient, volume) except Exception as e: - self.fail("Attach volume failed!") + self.fail("Attach volume failed with %s!" % e) volumes = Volume.list( self.apiclient, From 620eb509418bbd8c3c21932be0e89567eb0d705b Mon Sep 17 00:00:00 2001 From: Alena Prokharchyk Date: Thu, 1 Aug 2013 09:56:43 -0700 Subject: [PATCH 058/763] CLOUDSTACK-4015: don't insert the fake template record when upgrade from 304 to 305 --- .../cloud/upgrade/dao/Upgrade304to305.java | 7 +----- setup/db/db/vmwaretmplinsert-304to305.sql | 22 ------------------- 2 files changed, 1 insertion(+), 28 deletions(-) delete mode 100755 setup/db/db/vmwaretmplinsert-304to305.sql diff --git a/engine/schema/src/com/cloud/upgrade/dao/Upgrade304to305.java b/engine/schema/src/com/cloud/upgrade/dao/Upgrade304to305.java index f9b7e5ea94a..dc4668f5870 100644 --- a/engine/schema/src/com/cloud/upgrade/dao/Upgrade304to305.java +++ b/engine/schema/src/com/cloud/upgrade/dao/Upgrade304to305.java @@ -56,13 +56,8 @@ public class Upgrade304to305 extends Upgrade30xBase implements DbUpgrade { if (script == null) { throw new CloudRuntimeException("Unable to find db/schema-304to305.sql"); } - - String vmWareTemplateInsertScript = Script.findScript("", "db/vmwaretmplinsert-304to305.sql"); - if (vmWareTemplateInsertScript == null) { - throw new CloudRuntimeException("Unable to find db/vmwaretmplinsert-304to305.sql"); - } - return new File[] { new File(vmWareTemplateInsertScript), new File(script) }; + return new File[] { new File(script) }; } @Override diff --git a/setup/db/db/vmwaretmplinsert-304to305.sql b/setup/db/db/vmwaretmplinsert-304to305.sql deleted file mode 100755 index 469d937e067..00000000000 --- a/setup/db/db/vmwaretmplinsert-304to305.sql +++ /dev/null @@ -1,22 +0,0 @@ --- Licensed to the Apache Software Foundation (ASF) under one --- or more contributor license agreements. See the NOTICE file --- distributed with this work for additional information --- regarding copyright ownership. The ASF licenses this file --- to you under the Apache License, Version 2.0 (the --- "License"); you may not use this file except in compliances --- with the License. You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, --- software distributed under the License is distributed on an --- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY --- KIND, either express or implied. See the License for the --- specific language governing permissions and limitations --- under the License. - -#This file doesn't exist on 3.0.x branch. The fake template record is being inserted just because this template will never be used in 4.2 version of the code -INSERT IGNORE INTO `cloud`.`vm_template` (unique_name, name, public, featured, type, hvm, bits, url, format, created, account_id, checksum, display_text, enable_password, - enable_sshkey, guest_os_id, bootable, prepopulate, cross_zones, extractable, hypervisor_type, source_template_id, template_tag, sort_key) -VALUES ("systemvm-vmware-3.0.5", "systemvm-vmware-3.0.5", 1, 1, "USER", 0, 64, "fake url", "fake format", now(), 1, null, "fake text", 0, - 0, 99, 1, 0, 1, 0, "VMware", NULL, NULL, 0); From 4fb4afb88ad8281c88351ac9e766054d5f12e6e1 Mon Sep 17 00:00:00 2001 From: Bharat Kumar Date: Thu, 25 Jul 2013 01:18:21 +0530 Subject: [PATCH 059/763] Cloudstack-3753 Multiple VLAN range API need to accept a list rather than "add" or "remove" per command Reviewed-by: Alena Prokharchyk --- api/src/com/cloud/network/NetworkService.java | 2 +- .../network/UpdatePhysicalNetworkCmd.java | 8 +- .../com/cloud/dc/dao/DataCenterVnetDao.java | 2 +- .../cloud/dc/dao/DataCenterVnetDaoImpl.java | 24 +- .../cloud/network/dao/PhysicalNetworkVO.java | 2 +- .../com/cloud/network/NetworkServiceImpl.java | 350 +++++++++--------- .../cloud/network/MockNetworkManagerImpl.java | 2 +- .../network/UpdatePhysicalNetworkTest.java | 2 +- .../com/cloud/vpc/MockNetworkManagerImpl.java | 2 +- 9 files changed, 195 insertions(+), 199 deletions(-) diff --git a/api/src/com/cloud/network/NetworkService.java b/api/src/com/cloud/network/NetworkService.java index 59ccdbf754d..87fecb0f873 100755 --- a/api/src/com/cloud/network/NetworkService.java +++ b/api/src/com/cloud/network/NetworkService.java @@ -87,7 +87,7 @@ public interface NetworkService { Long startIndex, Long pageSize, String name); PhysicalNetwork updatePhysicalNetwork(Long id, String networkSpeed, List tags, - String newVnetRangeString, String state, String removeVlan); + String newVnetRangeString, String state); boolean deletePhysicalNetwork(Long id); diff --git a/api/src/org/apache/cloudstack/api/command/admin/network/UpdatePhysicalNetworkCmd.java b/api/src/org/apache/cloudstack/api/command/admin/network/UpdatePhysicalNetworkCmd.java index 6d37dd8a49b..06cf38dba3f 100644 --- a/api/src/org/apache/cloudstack/api/command/admin/network/UpdatePhysicalNetworkCmd.java +++ b/api/src/org/apache/cloudstack/api/command/admin/network/UpdatePhysicalNetworkCmd.java @@ -54,8 +54,6 @@ public class UpdatePhysicalNetworkCmd extends BaseAsyncCmd { @Parameter(name=ApiConstants.VLAN, type=CommandType.STRING, description="the VLAN for the physical network") private String vlan; - @Parameter(name=ApiConstants.REMOVE_VLAN, type = CommandType.STRING, description ="The vlan range we want to remove") - private String removevlan; ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// @@ -81,10 +79,6 @@ public class UpdatePhysicalNetworkCmd extends BaseAsyncCmd { return vlan; } - public String getRemoveVlan(){ - return removevlan; - } - ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// @@ -101,7 +95,7 @@ public class UpdatePhysicalNetworkCmd extends BaseAsyncCmd { @Override public void execute(){ - PhysicalNetwork result = _networkService.updatePhysicalNetwork(getId(),getNetworkSpeed(), getTags(), getVlan(), getState(), getRemoveVlan()); + PhysicalNetwork result = _networkService.updatePhysicalNetwork(getId(),getNetworkSpeed(), getTags(), getVlan(), getState()); PhysicalNetworkResponse response = _responseGenerator.createPhysicalNetworkResponse(result); response.setResponseName(getCommandName()); this.setResponseObject(response); diff --git a/engine/schema/src/com/cloud/dc/dao/DataCenterVnetDao.java b/engine/schema/src/com/cloud/dc/dao/DataCenterVnetDao.java index e2e6b795d35..de68e1e30b1 100644 --- a/engine/schema/src/com/cloud/dc/dao/DataCenterVnetDao.java +++ b/engine/schema/src/com/cloud/dc/dao/DataCenterVnetDao.java @@ -33,7 +33,7 @@ public interface DataCenterVnetDao extends GenericDao { public void delete(long physicalNetworkId); - public void deleteRange(Transaction txn, long dcId, long physicalNetworkId, int start, int end); + public void deleteVnets(Transaction txn, long dcId, long physicalNetworkId, List vnets); public void lockRange(long dcId, long physicalNetworkId, Integer start, Integer end); diff --git a/engine/schema/src/com/cloud/dc/dao/DataCenterVnetDaoImpl.java b/engine/schema/src/com/cloud/dc/dao/DataCenterVnetDaoImpl.java index ced2982cf9d..3ac2729853b 100755 --- a/engine/schema/src/com/cloud/dc/dao/DataCenterVnetDaoImpl.java +++ b/engine/schema/src/com/cloud/dc/dao/DataCenterVnetDaoImpl.java @@ -110,9 +110,10 @@ public class DataCenterVnetDaoImpl extends GenericDaoBase argument each string is a vlan. not a vlanRange. public void add(long dcId, long physicalNetworkId, List vnets) { String insertVnet = "INSERT INTO `cloud`.`op_dc_vnet_alloc` (vnet, data_center_id, physical_network_id) VALUES ( ?, ?, ?)"; @@ -133,15 +134,18 @@ public class DataCenterVnetDaoImpl extends GenericDaoBase argument each string is a vlan. not a vlanRange. + public void deleteVnets(Transaction txn, long dcId, long physicalNetworkId, List vnets) { + String deleteVnet = "DELETE FROM `cloud`.`op_dc_vnet_alloc` WHERE data_center_id=? AND physical_network_id=? AND taken IS NULL AND vnet=?"; try { PreparedStatement stmt = txn.prepareAutoCloseStatement(deleteVnet); - stmt.setLong(1,dcId); - stmt.setLong(2,physicalNetworkId); - stmt.setString(3,((Integer)start).toString()); - stmt.setString(4,((Integer)end).toString()); - stmt.execute(); + for (int i =0; i <= vnets.size()-1; i++) { + stmt.setLong(1,dcId); + stmt.setLong(2,physicalNetworkId); + stmt.setString(3, vnets.get(i)); + stmt.addBatch(); + } + stmt.executeBatch(); } catch (SQLException e) { throw new CloudRuntimeException("Exception caught adding vnet ", e); } diff --git a/engine/schema/src/com/cloud/network/dao/PhysicalNetworkVO.java b/engine/schema/src/com/cloud/network/dao/PhysicalNetworkVO.java index f68eee1de5c..684a6008e9d 100644 --- a/engine/schema/src/com/cloud/network/dao/PhysicalNetworkVO.java +++ b/engine/schema/src/com/cloud/network/dao/PhysicalNetworkVO.java @@ -206,7 +206,7 @@ public class PhysicalNetworkVO implements PhysicalNetwork { public List> getVnet() { List > vnetList = new ArrayList>(); if (vnet != null) { - String [] Temp = vnet.split(";"); + String [] Temp = vnet.split(","); String [] vnetSplit = null; for (String vnetRange : Temp){ vnetSplit = vnetRange.split("-"); diff --git a/server/src/com/cloud/network/NetworkServiceImpl.java b/server/src/com/cloud/network/NetworkServiceImpl.java index 7e6b581de2d..90368be0a92 100755 --- a/server/src/com/cloud/network/NetworkServiceImpl.java +++ b/server/src/com/cloud/network/NetworkServiceImpl.java @@ -2425,34 +2425,12 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService { throw new InvalidParameterException("Only one isolationMethod can be specified for a physical network at this time"); } - int vnetStart = 0; - int vnetEnd = 0; - long minVnet = MIN_VLAN_ID; - long maxVnet = MAX_VLAN_ID; - // Wondering why GRE doesn't check its vNet range here. While they check it in processVlanRange called by updatePhysicalNetwork. - if (vnetRange != null) { // Verify zone type if (zoneType == NetworkType.Basic || (zoneType == NetworkType.Advanced && zone.isSecurityGroupEnabled())) { throw new InvalidParameterValueException("Can't add vnet range to the physical network in the zone that supports " + zoneType + " network, Security Group enabled: " + zone.isSecurityGroupEnabled()); } - - String[] tokens = vnetRange.split("-"); - try { - vnetStart = Integer.parseInt(tokens[0]); - if (tokens.length == 1) { - vnetEnd = vnetStart; - } else { - vnetEnd = Integer.parseInt(tokens[1]); - } - } catch (NumberFormatException e) { - throw new InvalidParameterValueException("Please specify valid integers for the vlan range."); - } - if ((vnetStart > vnetEnd) || (vnetStart < minVnet) || (vnetEnd > maxVnet)) { - s_logger.warn("Invalid vnet range: start range:" + vnetStart + " end range:" + vnetEnd); - throw new InvalidParameterValueException("Vnet range should be between " + minVnet + "-" + maxVnet + " and start range should be lesser than or equal to end range"); - } } BroadcastDomainRange broadcastDomainRange = null; @@ -2491,14 +2469,8 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService { pNetwork = _physicalNetworkDao.persist(pNetwork); // Add vnet entries for the new zone if zone type is Advanced - - List vnets = new ArrayList(); - for (Integer i= vnetStart; i<= vnetEnd; i++ ) { - vnets.add(i.toString()); - } - if (vnetRange != null) { - _dcDao.addVnet(zone.getId(), pNetwork.getId(), vnets); + addOrRemoveVnets(vnetRange.split(","), pNetwork); } // add VirtualRouter as the default network service provider @@ -2548,7 +2520,7 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService { @Override @DB @ActionEvent(eventType = EventTypes.EVENT_PHYSICAL_NETWORK_UPDATE, eventDescription = "updating physical network", async = true) - public PhysicalNetwork updatePhysicalNetwork(Long id, String networkSpeed, List tags, String newVnetRange, String state, String removeVlan) { + public PhysicalNetwork updatePhysicalNetwork(Long id, String networkSpeed, List tags, String newVnetRange, String state) { // verify input parameters PhysicalNetworkVO network = _physicalNetworkDao.findById(id); @@ -2573,11 +2545,6 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService { } } - if (removeVlan != null){ - List tokens = processVlanRange(network,removeVlan); - removeVlanRange(network, tokens.get(0), tokens.get(1)); - } - if (tags != null && tags.size() > 1) { throw new InvalidParameterException("Unable to support more than one tag on network yet"); } @@ -2603,173 +2570,204 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService { network.setSpeed(networkSpeed); } - // Vnet range can be extended only - boolean AddVnet = true; - List> vnetsToAdd = new ArrayList>(); - - List tokens = null; - List add_Vnet = null; if (newVnetRange != null) { - tokens = processVlanRange(network, newVnetRange); - HashSet vnetsInDb = new HashSet(); - vnetsInDb.addAll(_datacneter_vnet.listVnetsByPhysicalNetworkAndDataCenter(network.getDataCenterId(), id)); - HashSet tempVnets = new HashSet(); - tempVnets.addAll(vnetsInDb); - for (Integer i = tokens.get(0); i <= tokens.get(1); i++) { - tempVnets.add(i.toString()); - } - tempVnets.removeAll(vnetsInDb); - if (tempVnets.isEmpty()) { - throw new InvalidParameterValueException("The vlan range you are trying to add already exists."); - } - vnetsInDb.addAll(tempVnets); - add_Vnet = new ArrayList(); - add_Vnet.addAll(tempVnets); - List sortedList = new ArrayList(vnetsInDb); - Collections.sort(sortedList, new Comparator() { - public int compare(String s1, String s2) { - return Integer.valueOf(s1).compareTo(Integer.valueOf(s2)); - } - }); - //build the vlan string form the allocated vlan list. - String vnetRange = ""; - String startvnet = sortedList.get(0); - String endvnet = ""; - for ( int i =0; i < sortedList.size()-1; i++ ) { - if (Integer.valueOf(sortedList.get(i+1)) - Integer.valueOf(sortedList.get(i)) > 1) { - endvnet = sortedList.get(i); - vnetRange=vnetRange + startvnet+"-"+endvnet+";"; - startvnet = sortedList.get(i+1); - } - } - endvnet = sortedList.get(sortedList.size()-1); - vnetRange=vnetRange + startvnet+"-"+endvnet+";"; - vnetRange = vnetRange.substring(0,vnetRange.length()-1); - network.setVnet(vnetRange); - } - Transaction txn = Transaction.currentTxn(); - txn.start(); - if (add_Vnet != null) { - s_logger.debug("Adding vnet range " + tokens.get(0).toString() + "-" + tokens.get(1).toString() + " for the physicalNetwork id= " + id + " and zone id=" + network.getDataCenterId() - + " as a part of updatePhysicalNetwork call"); - _dcDao.addVnet(network.getDataCenterId(), network.getId(), add_Vnet); + String [] listOfRanges = newVnetRange.split(","); + addOrRemoveVnets(listOfRanges, network); } _physicalNetworkDao.update(id, network); - txn.commit(); - return network; + } - private List processVlanRange(PhysicalNetworkVO network, String vlan) { + @DB + public void addOrRemoveVnets(String [] listOfRanges, PhysicalNetworkVO network) { + List addVnets = null; + List removeVnets =null; + HashSet tempVnets = new HashSet(); + HashSet vnetsInDb = new HashSet(); + List> vnetranges = null; + String comaSeperatedStingOfVnetRanges = null; + int i =0; + if (listOfRanges.length !=0) { + _physicalNetworkDao.acquireInLockTable(network.getId(),10); + vnetranges = validateVlanRange(network, listOfRanges); + + //computing vnets to be removed. + removeVnets = getVnetsToremove(network, vnetranges); + + //computing vnets to add + vnetsInDb.addAll(_datacneter_vnet.listVnetsByPhysicalNetworkAndDataCenter(network.getDataCenterId(), network.getId())); + tempVnets.addAll(vnetsInDb); + for (Pairvlan : vnetranges) { + for (i= vlan.first(); i<= vlan.second(); i++) { + tempVnets.add(Integer.toString(i)); + } + } + tempVnets.removeAll(vnetsInDb); + + //vnets to add in tempVnets. + //adding and removing vnets from vnetsInDb + if (removeVnets != null && removeVnets.size() !=0 ) { + vnetsInDb.removeAll(removeVnets); + } + + if (tempVnets.size() != 0) { + addVnets = new ArrayList(); + addVnets.addAll(tempVnets); + vnetsInDb.addAll(tempVnets); + } + + //sorting the vnets in Db to generate a coma seperated list of the vnet string. + if (vnetsInDb.size() !=0 ) { + comaSeperatedStingOfVnetRanges = generateVnetString( new ArrayList(vnetsInDb)); + } + network.setVnet(comaSeperatedStingOfVnetRanges); + + Transaction txn = Transaction.currentTxn(); + txn.start(); + if (addVnets != null) { + s_logger.debug("Adding vnet range " + addVnets.toString()+ " for the physicalNetwork id= " + network.getId() + " and zone id=" + network.getDataCenterId() + + " as a part of updatePhysicalNetwork call"); + //add vnet takes a list of strings to be added. each string is a vnet. + _dcDao.addVnet(network.getDataCenterId(), network.getId(), addVnets); + } + if (removeVnets != null) { + s_logger.debug("removing vnet range " + removeVnets.toString()+ " for the physicalNetwork id= " + network.getId() + " and zone id=" + network.getDataCenterId() + + " as a part of updatePhysicalNetwork call"); + //deleteVnets takes a list of strings to be removed. each string is a vnet. + _datacneter_vnet.deleteVnets(txn, network.getDataCenterId(), network.getId(), removeVnets); + } + _physicalNetworkDao.update(network.getId(), network); + txn.commit(); + _physicalNetworkDao.releaseFromLockTable(network.getId()); + } + } + + private List> validateVlanRange(PhysicalNetworkVO network, String[] listOfRanges) { Integer StartVnet; Integer EndVnet; - String[] VnetRange = vlan.split("-"); + List> vlanTokens = new ArrayList>(); + for (String vlanRange : listOfRanges) { + String[] VnetRange = vlanRange.split("-"); - // Init with [min,max] of VLAN. Actually 0x000 and 0xFFF are reserved by IEEE, shoudn't be used. - long minVnet = MIN_VLAN_ID; - long maxVnet = MAX_VLAN_ID; + // Init with [min,max] of VLAN. Actually 0x000 and 0xFFF are reserved by IEEE, shoudn't be used. + long minVnet = MIN_VLAN_ID; + long maxVnet = MAX_VLAN_ID; - // for GRE phynets allow up to 32bits - // TODO: Not happy about this test. - // What about guru-like objects for physical networs? - s_logger.debug("ISOLATION METHODS:" + network.getIsolationMethods()); - // Java does not have unsigned types... - if (network.getIsolationMethods().contains("GRE")) { - minVnet = MIN_GRE_KEY; - maxVnet = MAX_GRE_KEY; - } - String rangeMessage = " between " + minVnet + " and " + maxVnet; - if (VnetRange.length < 2) { - throw new InvalidParameterValueException("Please provide valid vnet range" + rangeMessage); - } + // for GRE phynets allow up to 32bits + // TODO: Not happy about this test. + // What about guru-like objects for physical networs? + s_logger.debug("ISOLATION METHODS:" + network.getIsolationMethods()); + // Java does not have unsigned types... + if (network.getIsolationMethods().contains("GRE")) { + minVnet = MIN_GRE_KEY; + maxVnet = MAX_GRE_KEY; + } + String rangeMessage = " between " + minVnet + " and " + maxVnet; + if (VnetRange.length == 1 && VnetRange[0].equals("")) { + return vlanTokens; + } + if (VnetRange.length < 2) { + throw new InvalidParameterValueException("Please provide valid vnet range. vnet range should be a coma seperated list of vlan ranges. example 500-500,600-601" + rangeMessage); + } - if (VnetRange[0] == null || VnetRange[1] == null) { - throw new InvalidParameterValueException("Please provide valid vnet range" + rangeMessage); - } + if (VnetRange[0] == null || VnetRange[1] == null) { + throw new InvalidParameterValueException("Please provide valid vnet range" + rangeMessage); + } - try { - StartVnet = Integer.parseInt(VnetRange[0]); - EndVnet = Integer.parseInt(VnetRange[1]); - } catch (NumberFormatException e) { - s_logger.warn("Unable to parse vnet range:", e); - throw new InvalidParameterValueException("Please provide valid vnet range" + rangeMessage); - } - if (StartVnet < minVnet || EndVnet > maxVnet) { - throw new InvalidParameterValueException("Vnet range has to be" + rangeMessage); - } + try { + StartVnet = Integer.parseInt(VnetRange[0]); + EndVnet = Integer.parseInt(VnetRange[1]); + } catch (NumberFormatException e) { + s_logger.warn("Unable to parse vnet range:", e); + throw new InvalidParameterValueException("Please provide valid vnet range. The vnet range should be a coma seperated list example 2001-2012,3000-3005." + rangeMessage); + } + if (StartVnet < minVnet || EndVnet > maxVnet) { + throw new InvalidParameterValueException("Vnet range has to be" + rangeMessage); + } - if (StartVnet > EndVnet) { - throw new InvalidParameterValueException("Vnet range has to be" + rangeMessage + " and start range should be lesser than or equal to stop range"); + if (StartVnet > EndVnet) { + throw new InvalidParameterValueException("Vnet range has to be" + rangeMessage + " and start range should be lesser than or equal to stop range"); + } + vlanTokens.add(new Pair(StartVnet, EndVnet)); } - List tokens = new ArrayList(); - tokens.add(StartVnet); - tokens.add(EndVnet); - return tokens; + return vlanTokens; } - private boolean removeVlanRange( PhysicalNetworkVO network, Integer start, Integer end) { - Integer temp=0; + public String generateVnetString(List vnetList) { + Collections.sort(vnetList, new Comparator() { + public int compare(String s1, String s2) { + return Integer.valueOf(s1).compareTo(Integer.valueOf(s2)); + } + }); int i; - List > existingRanges = network.getVnet(); - Transaction txn = Transaction.currentTxn(); - txn.start(); - _physicalNetworkDao.acquireInLockTable(network.getId(),10); - _datacneter_vnet.lockRange(network.getDataCenterId(), network.getId(), start, end); - List result = _datacneter_vnet.listAllocatedVnetsInRange(network.getDataCenterId(), network.getId(), start, end); - if (!result.isEmpty()){ - txn.close(); - throw new InvalidParameterValueException("Some of the vnets from this range are allocated, can only remove a range which has no allocated vnets"); - } - // If the range is partially dedicated to an account fail the request - List maps = _accountGuestVlanMapDao.listAccountGuestVlanMapsByPhysicalNetwork(network.getId()); - for (AccountGuestVlanMapVO map : maps) { - String[] vlans = map.getGuestVlanRange().split("-"); - Integer dedicatedStartVlan = Integer.parseInt(vlans[0]); - Integer dedicatedEndVlan = Integer.parseInt(vlans[1]); - if ((start >= dedicatedStartVlan && start <= dedicatedEndVlan) || (end >= dedicatedStartVlan && end <= dedicatedEndVlan)) { - txn.close(); - throw new InvalidParameterValueException("Vnet range " + map.getGuestVlanRange() + " is dedicated" + - " to an account. The specified range " + start + "-" + end + " overlaps with the dedicated range " + - " Please release the overlapping dedicated range before deleting the range"); + //build the vlan string form the sorted list. + String vnetRange = ""; + String startvnet = vnetList.get(0); + String endvnet = ""; + for ( i =0; i < vnetList.size()-1; i++ ) { + if (Integer.valueOf(vnetList.get(i+1)) - Integer.valueOf(vnetList.get(i)) > 1) { + endvnet = vnetList.get(i); + vnetRange=vnetRange + startvnet+"-"+endvnet+","; + startvnet = vnetList.get(i+1); } } - for (i=0; i= end){ - temp = existingRanges.get(i).second(); - existingRanges.get(i).second(start - 1); - existingRanges.add(new Pair((end+1),temp)); - break; + endvnet = vnetList.get(vnetList.size()-1); + vnetRange=vnetRange + startvnet+"-"+endvnet+","; + vnetRange=vnetRange.substring(0,vnetRange.length()-1); + return vnetRange; + } + + private List getVnetsToremove(PhysicalNetworkVO network, List> vnetRanges) { + int i; + List removeVnets = new ArrayList(); + HashSet vnetsInDb = new HashSet(); + vnetsInDb.addAll(_datacneter_vnet.listVnetsByPhysicalNetworkAndDataCenter(network.getDataCenterId(), network.getId())); + //remove all the vnets from vnets in db to check if there are any vnets that are not there in given list. + //remove all the vnets not in the list of vnets passed by the user. + if (vnetRanges.size() == 0) { + //this implies remove all vlans. + removeVnets.addAll(vnetsInDb); + return removeVnets; + } + for (Pairvlan : vnetRanges) { + for (i= vlan.first(); i<= vlan.second(); i++) { + vnetsInDb.remove(Integer.toString(i)); } } + String vnetRange = null; + if (vnetsInDb.size() != 0) { + removeVnets.addAll(vnetsInDb); + vnetRange = generateVnetString(removeVnets); + }else { + return removeVnets; + } - if (temp == 0){ - throw new InvalidParameterValueException("The vnet range you are trying to delete does not exist."); - } - if(existingRanges.get(i).first() > existingRanges.get(i).second()){ - existingRanges.remove(i); - } - if(existingRanges.get(existingRanges.size()-1).first() > existingRanges.get(existingRanges.size()-1).second()){ - existingRanges.remove(existingRanges.size()-1); - } - _datacneter_vnet.deleteRange(txn, network.getDataCenterId(), network.getId(), start, end); - - String vnetString=""; - if (existingRanges.isEmpty()) { - network.setVnet(null); - } else { - for (Pair vnetRange : existingRanges ) { - vnetString=vnetString+vnetRange.first().toString()+"-"+vnetRange.second().toString()+";"; + for (String vnet : vnetRange.split(",")) { + String [] range = vnet.split("-"); + Integer start = Integer.parseInt(range[0]); + Integer end= Integer.parseInt(range[1]); + _datacneter_vnet.lockRange(network.getDataCenterId(), network.getId(), start, end); + List result = _datacneter_vnet.listAllocatedVnetsInRange(network.getDataCenterId(), network.getId(), start, end); + if (!result.isEmpty()){ + throw new InvalidParameterValueException("Some of the vnets from this range are allocated, can only remove a range which has no allocated vnets"); + } + // If the range is partially dedicated to an account fail the request + List maps = _accountGuestVlanMapDao.listAccountGuestVlanMapsByPhysicalNetwork(network.getId()); + for (AccountGuestVlanMapVO map : maps) { + String[] vlans = map.getGuestVlanRange().split("-"); + Integer dedicatedStartVlan = Integer.parseInt(vlans[0]); + Integer dedicatedEndVlan = Integer.parseInt(vlans[1]); + if ((start >= dedicatedStartVlan && start <= dedicatedEndVlan) || (end >= dedicatedStartVlan && end <= dedicatedEndVlan)) { + throw new InvalidParameterValueException("Vnet range " + map.getGuestVlanRange() + " is dedicated" + + " to an account. The specified range " + start + "-" + end + " overlaps with the dedicated range " + + " Please release the overlapping dedicated range before deleting the range"); + } } - vnetString = vnetString.substring(0, vnetString.length()-1); - network.setVnet(vnetString); } - _physicalNetworkDao.update(network.getId(), network); - txn.commit(); - _physicalNetworkDao.releaseFromLockTable(network.getId()); - - return true; + return removeVnets; } diff --git a/server/test/com/cloud/network/MockNetworkManagerImpl.java b/server/test/com/cloud/network/MockNetworkManagerImpl.java index a3d6f53f1eb..7f34e5556d7 100755 --- a/server/test/com/cloud/network/MockNetworkManagerImpl.java +++ b/server/test/com/cloud/network/MockNetworkManagerImpl.java @@ -351,7 +351,7 @@ public class MockNetworkManagerImpl extends ManagerBase implements NetworkManage } @Override - public PhysicalNetwork updatePhysicalNetwork(Long id, String networkSpeed, List tags, String newVnetRangeString, String state, String removeVlan) { + public PhysicalNetwork updatePhysicalNetwork(Long id, String networkSpeed, List tags, String newVnetRangeString, String state) { // TODO Auto-generated method stub return null; } diff --git a/server/test/com/cloud/network/UpdatePhysicalNetworkTest.java b/server/test/com/cloud/network/UpdatePhysicalNetworkTest.java index e3fc36a05d2..1c0eff6453a 100644 --- a/server/test/com/cloud/network/UpdatePhysicalNetworkTest.java +++ b/server/test/com/cloud/network/UpdatePhysicalNetworkTest.java @@ -65,7 +65,7 @@ public class UpdatePhysicalNetworkTest { when(_datacenterDao.findById(anyLong())).thenReturn(datacentervo); when(_physicalNetworkDao.update(anyLong(), any(physicalNetworkVO.getClass()))).thenReturn(true); when(_DatacenterVnetDao.listVnetsByPhysicalNetworkAndDataCenter(anyLong(), anyLong())).thenReturn(existingRange); - networkService.updatePhysicalNetwork(1l, null, null, "525-530", null, null); + networkService.updatePhysicalNetwork(1l, null, null, "524-524,525-530", null); txn.close("updatePhysicalNetworkTest"); verify(physicalNetworkVO).setVnet(argumentCaptor.capture()); assertEquals("524-530", argumentCaptor.getValue()); diff --git a/server/test/com/cloud/vpc/MockNetworkManagerImpl.java b/server/test/com/cloud/vpc/MockNetworkManagerImpl.java index a28284084a4..178ebbad2a9 100644 --- a/server/test/com/cloud/vpc/MockNetworkManagerImpl.java +++ b/server/test/com/cloud/vpc/MockNetworkManagerImpl.java @@ -372,7 +372,7 @@ public class MockNetworkManagerImpl extends ManagerBase implements NetworkManage */ @Override public PhysicalNetwork updatePhysicalNetwork(Long id, String networkSpeed, List tags, - String newVnetRangeString, String state, String removeVlan) { + String newVnetRangeString, String state) { // TODO Auto-generated method stub return null; } From ef8e04a1a6ee39a6415c8418fedbac25f10a259f Mon Sep 17 00:00:00 2001 From: Jessica Wang Date: Thu, 1 Aug 2013 11:34:35 -0700 Subject: [PATCH 060/763] CLOUDSTACK-3344: UI > util > _s() should not include ampersand. --- ui/scripts/ui/utils.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ui/scripts/ui/utils.js b/ui/scripts/ui/utils.js index aeaf02babea..769aea7fa54 100644 --- a/ui/scripts/ui/utils.js +++ b/ui/scripts/ui/utils.js @@ -90,8 +90,7 @@ return ''; } - var sanitized = value - .replace(/&/g, "&") + var sanitized = value .replace(//g, ">"); From 84151fddd1528fb0f8d32da5e8f4a441491f54fe Mon Sep 17 00:00:00 2001 From: Jessica Wang Date: Thu, 1 Aug 2013 11:37:30 -0700 Subject: [PATCH 061/763] CLOUDSTACK-3344: revert e03f7f4db3c23eb5e4b72e62aa8fefd54f1c2355 --- ui/scripts/globalSettings.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/scripts/globalSettings.js b/ui/scripts/globalSettings.js index 947a3b27966..bee6ae3f489 100644 --- a/ui/scripts/globalSettings.js +++ b/ui/scripts/globalSettings.js @@ -298,7 +298,7 @@ array.push("&bindpass=" + todb(args.data.password)); array.push("&hostname=" + todb(args.data.hostname)); array.push("&searchbase=" + todb(args.data.searchbase)); - array.push("&queryfilter=" + todb(args.data.queryfilter).replace('&', '%26')); + array.push("&queryfilter=" + todb(args.data.queryfilter)); array.push("&port=" + todb(args.data.port)); if (args.$form.find('.form-item[rel=ssl]').find('input[type=checkbox]').is(':Checked') == true) { From 0a85c1d4a72506959a68a9d2faa12597da06f19a Mon Sep 17 00:00:00 2001 From: John Burwell Date: Thu, 1 Aug 2013 15:40:56 -0400 Subject: [PATCH 062/763] - CLOUSTACK-3229: Properly converts the Content-Length to string representation in s3xen --- scripts/vm/hypervisor/xenserver/s3xen | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/vm/hypervisor/xenserver/s3xen b/scripts/vm/hypervisor/xenserver/s3xen index 2798e2b05d2..372a6daaddc 100644 --- a/scripts/vm/hypervisor/xenserver/s3xen +++ b/scripts/vm/hypervisor/xenserver/s3xen @@ -270,7 +270,7 @@ class S3Client(object): headers = { self.HEADER_CONTENT_MD5: compute_md5(src_filename), self.HEADER_CONTENT_TYPE: 'application/octet-stream', - self.HEADER_CONTENT_LENGTH: os.stat(src_filename).st_size, + self.HEADER_CONTENT_LENGTH: str(os.stat(src_filename).st_size), } def send_body(connection): From 5384cbc41ddddffc652ee5f604823bfeb92e289c Mon Sep 17 00:00:00 2001 From: Edison Su Date: Thu, 1 Aug 2013 13:22:46 -0700 Subject: [PATCH 063/763] CLOUDSTACK-4008: return virtual size of volume in KVM --- .../com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java index 2f87ad49212..35cdaf34186 100644 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java @@ -270,7 +270,7 @@ public class KVMStorageProcessor implements StorageProcessor { VolumeObjectTO newVol = new VolumeObjectTO(); newVol.setPath(vol.getName()); - newVol.setSize(vol.getSize()); + newVol.setSize(volume.getSize()); return new CopyCmdAnswer(newVol); } catch (CloudRuntimeException e) { From 3c05999354526d6c31035de333df3a203a0f70a4 Mon Sep 17 00:00:00 2001 From: Jessica Wang Date: Thu, 1 Aug 2013 13:40:58 -0700 Subject: [PATCH 064/763] CLOUDSTACK-3713: UI > Infrastructure menu > System VMs > Scale Up action: call scaleSystemVm API instead of scaleVirtualMachine API. --- ui/scripts/system.js | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/ui/scripts/system.js b/ui/scripts/system.js index a156a375f38..15352f32d0b 100644 --- a/ui/scripts/system.js +++ b/ui/scripts/system.js @@ -6923,16 +6923,16 @@ action: function(args) { $.ajax({ - url: createURL("scaleVirtualMachine&id=" + args.context.systemVMs[0].id + "&serviceofferingid=" + args.data.serviceOfferingId), + url: createURL("scaleSystemVm&id=" + args.context.systemVMs[0].id + "&serviceofferingid=" + args.data.serviceOfferingId), dataType: "json", async: true, success: function(json) { - var jid = json.scalevirtualmachineresponse.jobid; + var jid = json.changeserviceforsystemvmresponse.jobid; args.response.success({ _custom: { jobId: jid, getUpdatedItem: function(json) { - return json.queryasyncjobresultresponse.jobresult.virtualmachine; + return json.queryasyncjobresultresponse.jobresult.systemvm; }, getActionFilter: function() { return systemvmActionfilter; @@ -7951,16 +7951,16 @@ action: function(args) { $.ajax({ - url: createURL("scaleVirtualMachine&id=" + args.context.routers[0].id + "&serviceofferingid=" + args.data.serviceOfferingId), + url: createURL("scaleSystemVm&id=" + args.context.routers[0].id + "&serviceofferingid=" + args.data.serviceOfferingId), dataType: "json", async: true, success: function(json) { - var jid = json.scalevirtualmachineresponse.jobid; + var jid = json.changeserviceforsystemvmresponse.jobid; args.response.success({ _custom: { jobId: jid, getUpdatedItem: function(json) { - return json.queryasyncjobresultresponse.jobresult.virtualmachine; + return json.queryasyncjobresultresponse.jobresult.systemvm; }, getActionFilter: function() { return routerActionfilter; @@ -8570,16 +8570,16 @@ action: function(args) { $.ajax({ - url: createURL("scaleVirtualMachine&id=" + args.context.systemVMs[0].id + "&serviceofferingid=" + args.data.serviceOfferingId), + url: createURL("scaleSystemVm&id=" + args.context.systemVMs[0].id + "&serviceofferingid=" + args.data.serviceOfferingId), dataType: "json", async: true, success: function(json) { - var jid = json.scalevirtualmachineresponse.jobid; + var jid = json.changeserviceforsystemvmresponse.jobid; args.response.success({ _custom: { jobId: jid, getUpdatedItem: function(json) { - return json.queryasyncjobresultresponse.jobresult.virtualmachine; + return json.queryasyncjobresultresponse.jobresult.systemvm; }, getActionFilter: function() { return vmActionfilter; From ebb30898c98dc538f9dffff970d74c597411d0ac Mon Sep 17 00:00:00 2001 From: Edison Su Date: Thu, 1 Aug 2013 13:57:47 -0700 Subject: [PATCH 065/763] CLOUDSTACK-4008: fix snapshot npe if its volume is deleted --- .../apache/cloudstack/storage/to/SnapshotObjectTO.java | 9 +++++++-- .../cloudstack/storage/volume/VolumeDataFactoryImpl.java | 3 +++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/engine/api/src/org/apache/cloudstack/storage/to/SnapshotObjectTO.java b/engine/api/src/org/apache/cloudstack/storage/to/SnapshotObjectTO.java index d2cb72a387c..b4829bf7aa4 100644 --- a/engine/api/src/org/apache/cloudstack/storage/to/SnapshotObjectTO.java +++ b/engine/api/src/org/apache/cloudstack/storage/to/SnapshotObjectTO.java @@ -22,6 +22,7 @@ import com.cloud.agent.api.to.DataObjectType; import com.cloud.agent.api.to.DataStoreTO; import com.cloud.agent.api.to.DataTO; import com.cloud.hypervisor.Hypervisor.HypervisorType; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; public class SnapshotObjectTO implements DataTO { private String path; @@ -40,8 +41,12 @@ public class SnapshotObjectTO implements DataTO { public SnapshotObjectTO(SnapshotInfo snapshot) { this.path = snapshot.getPath(); this.setId(snapshot.getId()); - this.volume = (VolumeObjectTO) snapshot.getBaseVolume().getTO(); - this.setVmName(snapshot.getBaseVolume().getAttachedVmName()); + VolumeInfo vol = snapshot.getBaseVolume(); + if (vol != null) { + this.volume = (VolumeObjectTO)vol.getTO(); + this.setVmName(vol.getAttachedVmName()); + } + SnapshotInfo parentSnapshot = snapshot.getParent(); if (parentSnapshot != null) { this.parentSnapshotPath = parentSnapshot.getPath(); diff --git a/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeDataFactoryImpl.java b/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeDataFactoryImpl.java index 2512c49348c..1d75ba1529b 100644 --- a/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeDataFactoryImpl.java +++ b/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeDataFactoryImpl.java @@ -74,6 +74,9 @@ public class VolumeDataFactoryImpl implements VolumeDataFactory { @Override public VolumeInfo getVolume(long volumeId) { VolumeVO volumeVO = volumeDao.findById(volumeId); + if (volumeVO == null) { + return null; + } VolumeObject vol = null; if (volumeVO.getPoolId() == null) { DataStore store = null; From ad9be8046f4574de483da3fc6a88c74264c8110a Mon Sep 17 00:00:00 2001 From: frank Date: Thu, 1 Aug 2013 14:55:46 -0700 Subject: [PATCH 066/763] CloudStack CLOUDSTACK-3986 baremetal - create basic zone for baremetal fail - Dhcp service provider for this network does not support the dhcp across multiple subnets --- .../cloud/baremetal/networkservice/BaremetalDhcpElement.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/hypervisors/baremetal/src/com/cloud/baremetal/networkservice/BaremetalDhcpElement.java b/plugins/hypervisors/baremetal/src/com/cloud/baremetal/networkservice/BaremetalDhcpElement.java index 96d702d7501..34e83027da7 100755 --- a/plugins/hypervisors/baremetal/src/com/cloud/baremetal/networkservice/BaremetalDhcpElement.java +++ b/plugins/hypervisors/baremetal/src/com/cloud/baremetal/networkservice/BaremetalDhcpElement.java @@ -65,7 +65,8 @@ public class BaremetalDhcpElement extends AdapterBase implements DhcpServiceProv static { Capability cap = new Capability(BaremetalDhcpManager.BAREMETAL_DHCP_SERVICE_CAPABITLITY); Map baremetalCaps = new HashMap(); - baremetalCaps.put(cap, null); + baremetalCaps.put(cap, null); + baremetalCaps.put(Capability.DhcpAccrossMultipleSubnets, Boolean.TRUE.toString()); capabilities = new HashMap>(); capabilities.put(Service.Dhcp, baremetalCaps); } From 17511b6a35427953b06119639ac34f5a0946bf4c Mon Sep 17 00:00:00 2001 From: Anthony Xu Date: Thu, 1 Aug 2013 15:31:26 -0700 Subject: [PATCH 067/763] CLOUDSTACK-4017: try to remove VDI host lock before start the VM which is using this VDI --- .../hypervisor/xen/resource/CitrixResourceBase.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java b/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java index 49433f86e89..e100211e978 100644 --- a/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java +++ b/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java @@ -1199,7 +1199,15 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe Volume.Type type = volume.getType(); VDI vdi = mount(conn, vmName, volume); - + if ( vdi != null ) { + Map smConfig = vdi.getSmConfig(conn); + for (String key : smConfig.keySet()) { + if (key.startsWith("host_")) { + vdi.removeFromSmConfig(conn, key); + break; + } + } + } VBD.Record vbdr = new VBD.Record(); vbdr.VM = vm; if (vdi != null) { From d77fc8093b37caf2259dd8aea93b447518cdb3c7 Mon Sep 17 00:00:00 2001 From: Jessica Wang Date: Thu, 1 Aug 2013 15:35:33 -0700 Subject: [PATCH 068/763] CLOUDSTACK-3753: UI > Infrastructure > zone detail > physical network > guest > (1) remove Delete VLAN Range action. (2) change Add VLAN Range action to pass new VLAN range + exisitng VLAN range(s) to API. This is corresponding UI change to updatePhysicalNetwork API change that removevlan property has been removed and definition of vlan property has been changed. --- ui/scripts/system.js | 80 ++++---------------------------------------- 1 file changed, 6 insertions(+), 74 deletions(-) diff --git a/ui/scripts/system.js b/ui/scripts/system.js index 15352f32d0b..6453dacdbb4 100644 --- a/ui/scripts/system.js +++ b/ui/scripts/system.js @@ -1151,11 +1151,13 @@ }, action: function(args) { - - var array1 = []; + var array1 = []; if (args.data.startvlan != "" && args.data.endvlan != "") { - array1.push("&vlan=" + todb(args.data.startvlan) + "-" + todb(args.data.endvlan)); - + var newVlanRange = todb(args.data.startvlan) + "-" + todb(args.data.endvlan); + if(selectedPhysicalNetworkObj.vlan == "") + array1.push("&vlan=" + newVlanRange); + else + array1.push("&vlan=" + selectedPhysicalNetworkObj.vlan + "," + newVlanRange); } $.ajax({ url: createURL("updatePhysicalNetwork&id=" + selectedPhysicalNetworkObj.id + array1.join("")), @@ -1168,85 +1170,15 @@ } }); }, - error: function(json) { args.response.error(parseXMLHttpResponse(json)); - } - }); - - }, notification: { poll: pollAsyncJobResult } - - - }, - - removeVlanRange: { - label: 'Remove VLAN Range', - messages: { - confirm: function(args) { - return 'Are you sure you want to remove an existing VLAN Range from this guest network?'; - }, - notification: function(args) { - return 'VLAN Range removed'; - } - }, - - createForm: { - title: 'Remove VLAN Range', - fields: { - startvlan: { - label: 'Vlan Start', - validation: { - required: true - } - }, - endvlan: { - label: 'Vlan End', - validation: { - required: true - } - } - } - - }, - - action: function(args) { - - var array1 = []; - if (args.data.startvlan != "" && args.data.endvlan != "") { - array1.push("&removevlan=" + args.data.startvlan + "-" + args.data.endvlan); - } - $.ajax({ - url: createURL("updatePhysicalNetwork&id=" + selectedPhysicalNetworkObj.id + array1.join("")), - dataType: "json", - success: function(json) { - var jobId = json.updatephysicalnetworkresponse.jobid; - args.response.success({ - _custom: { - jobId: jobId - } - }); - }, - - error: function(json) { - args.response.error(parseXMLHttpResponse(json)); - - } - - }); - - }, - notification: { - poll: pollAsyncJobResult - } - } - }, tabFilter: function(args) { From a059f596ac1ad04b8f948f66405807928bc7707c Mon Sep 17 00:00:00 2001 From: Jessica Wang Date: Thu, 1 Aug 2013 15:55:05 -0700 Subject: [PATCH 069/763] CLOUDSTACK-3753: UI > Infrastructure > zone > physical network > Guest > Edit action - make VLAN Range(s) editable. This is corresponding UI change to recent change of updatePhysicalNetwork API. --- ui/scripts/system.js | 42 +++++++++++++++++++----------------------- 1 file changed, 19 insertions(+), 23 deletions(-) diff --git a/ui/scripts/system.js b/ui/scripts/system.js index 6453dacdbb4..6ef832cb5b9 100644 --- a/ui/scripts/system.js +++ b/ui/scripts/system.js @@ -1082,22 +1082,23 @@ actions: { edit: { label: 'label.edit', - action: function(args) { - var vlan; - if (args.data.endVlan == null || args.data.endVlan.length == 0) - vlan = args.data.startVlan; - else - vlan = args.data.startVlan + "-" + args.data.endVlan; - - var array1 = []; - if (vlan != null && vlan.length > 0) - array1.push("&vlan=" + todb(vlan)); - if (args.data.tags != null && args.data.tags.length > 0) - array1.push("&tags=" + todb(args.data.tags)); - + action: function(args) { + var data = { + id: selectedPhysicalNetworkObj.id, + }; + if (args.data.vlan != null && args.data.vlan.length > 0) { + $.extend(data, { + vlan: args.data.vlan + }); + } + if (args.data.tags != null && args.data.tags.length > 0) { + $.extend(data, { + tags: args.data.tags + }); + } $.ajax({ - url: createURL("updatePhysicalNetwork&id=" + selectedPhysicalNetworkObj.id + array1.join("")), - dataType: "json", + url: createURL('updatePhysicalNetwork'), + data: data, success: function(json) { var jobId = json.updatephysicalnetworkresponse.jobid; @@ -1208,14 +1209,9 @@ label: 'label.state' }, vlan: { - label: 'VLAN Range(s)' - // isEditable: true - }, - /* endVlan: { - label: 'label.end.vlan', - isEditable: true - },*/ - + label: 'VLAN Range(s)', + isEditable: true + }, tags: { label: 'Tags', isEditable: true From 59fedb1bdc76837fc1a1d83bfcb12580d870869d Mon Sep 17 00:00:00 2001 From: Alena Prokharchyk Date: Thu, 1 Aug 2013 15:34:55 -0700 Subject: [PATCH 070/763] CLOUDSTACK-4020: lock nic entry in releaseNic method. Otherwise multiple threads can try to release the same nic at the same time, and it will lead to NPEs and backend failures --- .../com/cloud/network/NetworkManagerImpl.java | 13 ++- .../cloud/vm/VirtualMachineManagerImpl.java | 82 ++++++++++++------- 2 files changed, 60 insertions(+), 35 deletions(-) diff --git a/server/src/com/cloud/network/NetworkManagerImpl.java b/server/src/com/cloud/network/NetworkManagerImpl.java index d2761b0184f..b5a1ca77e66 100755 --- a/server/src/com/cloud/network/NetworkManagerImpl.java +++ b/server/src/com/cloud/network/NetworkManagerImpl.java @@ -2313,33 +2313,32 @@ public class NetworkManagerImpl extends ManagerBase implements NetworkManager, L ConcurrentOperationException, ResourceUnavailableException { List nics = _nicDao.listByVmId(vmProfile.getId()); for (NicVO nic : nics) { - releaseNic(vmProfile, nic); + releaseNic(vmProfile, nic.getId()); } } - + @Override @DB public void releaseNic(VirtualMachineProfile vmProfile, Nic nic) throws ConcurrentOperationException, ResourceUnavailableException { - NicVO nicVO = _nicDao.findById(nic.getId()); - releaseNic(vmProfile, nicVO); + releaseNic(vmProfile, nic.getId()); } @DB - protected void releaseNic(VirtualMachineProfile vmProfile, NicVO nicVO) + protected void releaseNic(VirtualMachineProfile vmProfile, long nicId) throws ConcurrentOperationException, ResourceUnavailableException { //lock the nic Transaction txn = Transaction.currentTxn(); txn.start(); - NicVO nic = _nicDao.lockRow(nicVO.getId(), true); + NicVO nic = _nicDao.lockRow(nicId, true); if (nic == null) { throw new ConcurrentOperationException("Unable to acquire lock on nic " + nic); } Nic.State originalState = nic.getState(); - NetworkVO network = _networksDao.findById(nicVO.getNetworkId()); + NetworkVO network = _networksDao.findById(nic.getNetworkId()); if (originalState == Nic.State.Reserved || originalState == Nic.State.Reserving) { if (nic.getReservationStrategy() == Nic.ReservationStrategy.Start) { diff --git a/server/src/com/cloud/vm/VirtualMachineManagerImpl.java b/server/src/com/cloud/vm/VirtualMachineManagerImpl.java index eedf4d20498..6d355392915 100755 --- a/server/src/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/server/src/com/cloud/vm/VirtualMachineManagerImpl.java @@ -3071,6 +3071,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } @Override + @DB public boolean removeVmFromNetwork(VirtualMachine vm, Network network, URI broadcastUri) throws ConcurrentOperationException, ResourceUnavailableException { VMInstanceVO vmVO = _vmDao.findById(vm.getId()); ReservationContext context = new ReservationContextImpl(null, null, _accountMgr.getActiveUser(User.UID_SYSTEM), @@ -3087,53 +3088,78 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac VirtualMachineTO vmTO = hvGuru.implement(vmProfile); Nic nic = null; - if (broadcastUri != null) { nic = _nicsDao.findByNetworkIdInstanceIdAndBroadcastUri(network.getId(), vm.getId(), broadcastUri.toString()); } else { nic = _networkModel.getNicInNetwork(vm.getId(), network.getId()); } - + if (nic == null){ s_logger.warn("Could not get a nic with " + network); return false; } - + // don't delete default NIC on a user VM if (nic.isDefaultNic() && vm.getType() == VirtualMachine.Type.User ) { s_logger.warn("Failed to remove nic from " + vm + " in " + network + ", nic is default."); throw new CloudRuntimeException("Failed to remove nic from " + vm + " in " + network + ", nic is default."); } - NicProfile nicProfile = new NicProfile(nic, network, nic.getBroadcastUri(), nic.getIsolationUri(), - _networkModel.getNetworkRate(network.getId(), vm.getId()), - _networkModel.isSecurityGroupSupportedInNetwork(network), - _networkModel.getNetworkTag(vmProfile.getVirtualMachine().getHypervisorType(), network)); - - //1) Unplug the nic - if (vm.getState() == State.Running) { - NicTO nicTO = toNicTO(nicProfile, vmProfile.getVirtualMachine().getHypervisorType()); - s_logger.debug("Un-plugging nic for vm " + vm + " from network " + network); - boolean result = vmGuru.unplugNic(network, nicTO, vmTO, context, dest); - if (result) { - s_logger.debug("Nic is unplugged successfully for vm " + vm + " in network " + network ); - } else { - s_logger.warn("Failed to unplug nic for the vm " + vm + " from network " + network); - return false; + //Lock on nic is needed here + Nic lock = _nicsDao.acquireInLockTable(nic.getId()); + if (lock == null) { + //check if nic is still there. Return if it was released already + if (_nicsDao.findById(nic.getId()) == null) { + if (s_logger.isDebugEnabled()) { + s_logger.debug("Not need to remove the vm " + vm + " from network " + network + " as the vm doesn't have nic in this network"); + } + return true; } - } else if (vm.getState() != State.Stopped) { - s_logger.warn("Unable to remove vm " + vm + " from network " + network); - throw new ResourceUnavailableException("Unable to remove vm " + vm + " from network, is not in the right state", - DataCenter.class, vm.getDataCenterId()); + throw new ConcurrentOperationException("Unable to lock nic " + nic.getId()); } + + if (s_logger.isDebugEnabled()) { + s_logger.debug("Lock is acquired for nic id " + lock.getId() + " as a part of remove vm " + vm + " from network " + network); + } + + try { + NicProfile nicProfile = new NicProfile(nic, network, nic.getBroadcastUri(), nic.getIsolationUri(), + _networkModel.getNetworkRate(network.getId(), vm.getId()), + _networkModel.isSecurityGroupSupportedInNetwork(network), + _networkModel.getNetworkTag(vmProfile.getVirtualMachine().getHypervisorType(), network)); - //2) Release the nic - _networkMgr.releaseNic(vmProfile, nic); - s_logger.debug("Successfully released nic " + nic + "for vm " + vm); + //1) Unplug the nic + if (vm.getState() == State.Running) { + NicTO nicTO = toNicTO(nicProfile, vmProfile.getVirtualMachine().getHypervisorType()); + s_logger.debug("Un-plugging nic for vm " + vm + " from network " + network); + boolean result = vmGuru.unplugNic(network, nicTO, vmTO, context, dest); + if (result) { + s_logger.debug("Nic is unplugged successfully for vm " + vm + " in network " + network ); + } else { + s_logger.warn("Failed to unplug nic for the vm " + vm + " from network " + network); + return false; + } + } else if (vm.getState() != State.Stopped) { + s_logger.warn("Unable to remove vm " + vm + " from network " + network); + throw new ResourceUnavailableException("Unable to remove vm " + vm + " from network, is not in the right state", + DataCenter.class, vm.getDataCenterId()); + } - //3) Remove the nic - _networkMgr.removeNic(vmProfile, nic); - return true; + //2) Release the nic + _networkMgr.releaseNic(vmProfile, nic); + s_logger.debug("Successfully released nic " + nic + "for vm " + vm); + + //3) Remove the nic + _networkMgr.removeNic(vmProfile, nic); + return true; + } finally { + if (lock != null) { + _nicsDao.releaseFromLockTable(lock.getId()); + if (s_logger.isDebugEnabled()) { + s_logger.debug("Lock is released for nic id " + lock.getId() + " as a part of remove vm " + vm + " from network " + network); + } + } + } } @Override From d9cc453240c842ab8aae58a91da20e4aa294a845 Mon Sep 17 00:00:00 2001 From: Jessica Wang Date: Thu, 1 Aug 2013 16:40:30 -0700 Subject: [PATCH 071/763] CLOUDSTACK-3753: UI > Infrastructure > zone > physical network > Guest > remove Add VLAN Range action since VLAN Range(s) is editable through Edit action. --- ui/scripts/system.js | 62 -------------------------------------------- 1 file changed, 62 deletions(-) diff --git a/ui/scripts/system.js b/ui/scripts/system.js index 6ef832cb5b9..3251f47b3ba 100644 --- a/ui/scripts/system.js +++ b/ui/scripts/system.js @@ -1117,68 +1117,6 @@ notification: { poll: pollAsyncJobResult } - }, - - addVlanRange: { - label: 'Add VLAN Range', - title: 'Add VLAN Range', - - messages: { - confirm: function(args) { - return 'Are you sure you want to add another VLAN Range to this guest network?'; - }, - notification: function(args) { - return 'VLAN Range added'; - } - }, - - createForm: { - title: 'Add VLAN Range', - fields: { - startvlan: { - label: 'Vlan Start', - validation: { - required: true - } - }, - endvlan: { - label: 'Vlan End', - validation: { - required: true - } - } - } - - }, - - action: function(args) { - var array1 = []; - if (args.data.startvlan != "" && args.data.endvlan != "") { - var newVlanRange = todb(args.data.startvlan) + "-" + todb(args.data.endvlan); - if(selectedPhysicalNetworkObj.vlan == "") - array1.push("&vlan=" + newVlanRange); - else - array1.push("&vlan=" + selectedPhysicalNetworkObj.vlan + "," + newVlanRange); - } - $.ajax({ - url: createURL("updatePhysicalNetwork&id=" + selectedPhysicalNetworkObj.id + array1.join("")), - dataType: "json", - success: function(json) { - var jobId = json.updatephysicalnetworkresponse.jobid; - args.response.success({ - _custom: { - jobId: jobId - } - }); - }, - error: function(json) { - args.response.error(parseXMLHttpResponse(json)); - } - }); - }, - notification: { - poll: pollAsyncJobResult - } } }, From 4ccbbd029d6f9bad43bda20f58fc8157ee90bded Mon Sep 17 00:00:00 2001 From: Min Chen Date: Thu, 1 Aug 2013 17:04:40 -0700 Subject: [PATCH 072/763] CLOUDSTACK-4002: [Upgrade][2.2.16 to 4.2] After upgrade creation of volume from a snapshot which was existing before upgrade, fails with NPE --- .../storage/datastore/db/SnapshotDataStoreVO.java | 2 +- .../storage/datastore/db/TemplateDataStoreVO.java | 2 +- .../cloudstack/storage/datastore/db/VolumeDataStoreVO.java | 2 +- .../schema/src/com/cloud/upgrade/dao/Upgrade410to420.java | 6 +++--- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/engine/api/src/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreVO.java b/engine/api/src/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreVO.java index 300df1e9673..0fe5e088043 100644 --- a/engine/api/src/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreVO.java +++ b/engine/api/src/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreVO.java @@ -193,7 +193,7 @@ public class SnapshotDataStoreVO implements StateObject Date: Thu, 1 Aug 2013 17:24:22 -0700 Subject: [PATCH 073/763] CLOUDSTACK-3979: fix deleta.snapshot.max issue --- .../snapshot/XenserverSnapshotStrategy.java | 68 +++++++++++-------- 1 file changed, 41 insertions(+), 27 deletions(-) diff --git a/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/XenserverSnapshotStrategy.java b/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/XenserverSnapshotStrategy.java index 10f3a4c5fdd..d9ee75ee816 100644 --- a/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/XenserverSnapshotStrategy.java +++ b/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/XenserverSnapshotStrategy.java @@ -58,11 +58,43 @@ public class XenserverSnapshotStrategy extends SnapshotStrategyBase { @Inject SnapshotDataFactory snapshotDataFactory; + protected boolean needFullBackup(SnapshotInfo snapshot) { + if (snapshot == null) { + return true; + } + boolean fullBackup = false; + int _deltaSnapshotMax = NumbersUtil.parseInt(configDao.getValue("snapshot.delta.max"), + SnapshotManager.DELTAMAX); + int deltaSnap = _deltaSnapshotMax; + SnapshotDataStoreVO parentSnapshotOnBackupStore = this.snapshotStoreDao.findBySnapshot(snapshot.getId(), DataStoreRole.Image); + int i; + for (i = 1; (i < deltaSnap && (parentSnapshotOnBackupStore != null)); i++) { + Long prevBackupId = parentSnapshotOnBackupStore.getParentSnapshotId(); + + if (prevBackupId == 0 || prevBackupId == null) { + break; + } + + parentSnapshotOnBackupStore = this.snapshotStoreDao.findBySnapshot(prevBackupId, DataStoreRole.Image); + } + if (i >= deltaSnap) { + fullBackup = true; + } + return fullBackup; + } + + protected boolean isEmptySnapshot(SnapshotInfo snapshotInfo, SnapshotInfo parentSnapshot) { + if (parentSnapshot != null && snapshotInfo.getPath().equalsIgnoreCase(parentSnapshot.getPath())) { + return true; + } else { + return false; + } + } @Override public SnapshotInfo backupSnapshot(SnapshotInfo snapshot) { SnapshotInfo parentSnapshot = snapshot.getParent(); - - if (parentSnapshot != null && snapshot.getPath().equalsIgnoreCase(parentSnapshot.getPath())) { + boolean fullBackup = needFullBackup(parentSnapshot); + if (!fullBackup && isEmptySnapshot(snapshot, parentSnapshot)) { s_logger.debug("backup an empty snapshot"); // don't need to backup this snapshot SnapshotDataStoreVO parentSnapshotOnBackupStore = this.snapshotStoreDao.findBySnapshot( @@ -93,33 +125,15 @@ public class XenserverSnapshotStrategy extends SnapshotStrategyBase { } } - // determine full snapshot backup or not - - boolean fullBackup = false; - - if (parentSnapshot != null) { - int _deltaSnapshotMax = NumbersUtil.parseInt(configDao.getValue("snapshot.delta.max"), - SnapshotManager.DELTAMAX); - int deltaSnap = _deltaSnapshotMax; - - SnapshotDataStoreVO parentSnapshotOnBackupStore = this.snapshotStoreDao.findBySnapshot(parentSnapshot.getId(), - DataStoreRole.Image); - int i; - for (i = 1; (i < deltaSnap && (parentSnapshotOnBackupStore != null)); i++) { - Long prevBackupId = parentSnapshotOnBackupStore.getParentSnapshotId(); - - if (prevBackupId == 0) { - break; - } - - parentSnapshotOnBackupStore = this.snapshotStoreDao.findBySnapshot(prevBackupId, DataStoreRole.Image); - } - if (i >= deltaSnap) { - fullBackup = true; + //if fullbackup, delete the snapshot on primary storage in db, so that, snapshot.getparent will return empty, thus full snapshot on the backend + if (fullBackup && parentSnapshot != null) { + s_logger.debug("reach delta snapshots max, delete the snapshot on primary storage in db"); + SnapshotDataStoreVO parentSnapshotOnPrimary = snapshotStoreDao.findByStoreSnapshot(DataStoreRole.Primary, parentSnapshot.getDataStore().getId(), + parentSnapshot.getId()); + if (parentSnapshotOnPrimary != null) { + snapshotStoreDao.remove(parentSnapshotOnPrimary.getId()); } } - - snapshot.addPayload(fullBackup); return this.snapshotSvr.backupSnapshot(snapshot); } From 1294a13f43ee0e6fb1fb2e04136ad5f37ecf7d8d Mon Sep 17 00:00:00 2001 From: Fang Wang Date: Thu, 25 Jul 2013 16:03:56 -0700 Subject: [PATCH 074/763] CLOUDSTACK-3759 [Automation] Failed to attach volume to VM, while vm are in stopped state Fix the null pointer. Signed-off-by: Edison Su --- server/src/com/cloud/storage/VolumeManagerImpl.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/src/com/cloud/storage/VolumeManagerImpl.java b/server/src/com/cloud/storage/VolumeManagerImpl.java index d8cbf112126..abbc2f0f258 100644 --- a/server/src/com/cloud/storage/VolumeManagerImpl.java +++ b/server/src/com/cloud/storage/VolumeManagerImpl.java @@ -1543,6 +1543,10 @@ public class VolumeManagerImpl extends ManagerBase implements VolumeManager { } private boolean needMoveVolume(VolumeVO rootVolumeOfVm, VolumeInfo volume) { + if (rootVolumeOfVm.getPoolId() == null || volume.getPoolId() == null) { + return false; + } + DataStore storeForRootVol = dataStoreMgr.getPrimaryDataStore(rootVolumeOfVm.getPoolId()); DataStore storeForDataVol = dataStoreMgr.getPrimaryDataStore(volume.getPoolId()); From a3292231ddddffe2c564d1318b111d40a4168138 Mon Sep 17 00:00:00 2001 From: Sheng Yang Date: Thu, 1 Aug 2013 17:29:35 -0700 Subject: [PATCH 075/763] Automation: Reconnect the ssh for test_vpc_vm_life_cycle.py Since VM are reused for test cases, we need to reconnect the SSH client to avoid error like: Failed to SSH into VM, SSH session not active --- .../component/test_vpc_vm_life_cycle.py | 51 ++++++++++++++----- 1 file changed, 39 insertions(+), 12 deletions(-) diff --git a/test/integration/component/test_vpc_vm_life_cycle.py b/test/integration/component/test_vpc_vm_life_cycle.py index ed86ae35615..3531affadf2 100644 --- a/test/integration/component/test_vpc_vm_life_cycle.py +++ b/test/integration/component/test_vpc_vm_life_cycle.py @@ -27,6 +27,8 @@ from marvin.integration.lib.base import * from marvin.integration.lib.common import * from marvin.remoteSSHClient import remoteSSHClient +import time + class Services: """Test VM life cycle in VPC network services """ @@ -426,7 +428,8 @@ class TestVMLifeCycleVPC(cloudstackTestCase): self.debug("Checking if we can SSH into VM_1 through %s?" % (self.public_ip_1.ipaddress.ipaddress)) ssh_1 = self.vm_1.get_ssh_client( - ipaddress=self.public_ip_1.ipaddress.ipaddress) + ipaddress=self.public_ip_1.ipaddress.ipaddress, + reconnect=True) self.debug("SSH into VM is successfully") self.debug("Verifying if we can ping to outside world from VM?") @@ -452,7 +455,8 @@ class TestVMLifeCycleVPC(cloudstackTestCase): (self.public_ip_2.ipaddress.ipaddress)) try: ssh_2 = self.vm_1.get_ssh_client( - ipaddress=self.public_ip_2.ipaddress.ipaddress) + ipaddress=self.public_ip_2.ipaddress.ipaddress, + reconnect=True) self.debug("SSH into VM is successfully") self.debug("Verifying if we can ping to outside world from VM?") @@ -563,6 +567,8 @@ class TestVMLifeCycleVPC(cloudstackTestCase): self.vm_2.start(self.apiclient) except Exception as e: self.fail("Failed to start the virtual instances, %s" % e) + # Wait until vms are up + time.sleep(120) self.debug("Validating if the network rules work properly or not?") self.validate_network_rules() return @@ -590,6 +596,8 @@ class TestVMLifeCycleVPC(cloudstackTestCase): except Exception as e: self.fail("Failed to reboot the virtual instances, %s" % e) + # Wait until vms are up + time.sleep(120) self.debug("Validating if the network rules work properly or not?") self.validate_network_rules() return @@ -666,6 +674,9 @@ class TestVMLifeCycleVPC(cloudstackTestCase): except Exception as e: self.fail("Failed to start the instances, %s" % e) + # Wait until vms are up + time.sleep(120) + self.debug("Validating if the network rules work properly or not?") self.validate_network_rules() return @@ -736,7 +747,8 @@ class TestVMLifeCycleVPC(cloudstackTestCase): try: ssh = self.vm_1.get_ssh_client( - ipaddress=self.public_ip_1.ipaddress.ipaddress) + ipaddress=self.public_ip_1.ipaddress.ipaddress, + reconnect=True) self.debug("SSH into VM is successfully") except Exception as e: self.fail("Failed to SSH into instance") @@ -790,7 +802,8 @@ class TestVMLifeCycleVPC(cloudstackTestCase): try: ssh = self.vm_1.get_ssh_client( - ipaddress=self.public_ip_1.ipaddress.ipaddress) + ipaddress=self.public_ip_1.ipaddress.ipaddress, + reconnect=True) self.debug("SSH into VM is successfully") except Exception as e: self.fail("Failed to SSH into instance") @@ -1151,8 +1164,11 @@ class TestVMLifeCycleSharedNwVPC(cloudstackTestCase): """Validating if the network rules (PF/LB) works properly or not?""" try: + self.debug("Checking if we can SSH into VM_1 through %s?" % + (self.public_ip_1.ipaddress.ipaddress)) ssh_1 = self.vm_1.get_ssh_client( - ipaddress=self.public_ip_1.ipaddress.ipaddress) + ipaddress=self.public_ip_1.ipaddress.ipaddress, + reconnect=True) self.debug("SSH into VM is successfully") self.debug("Verifying if we can ping to outside world from VM?") @@ -1515,7 +1531,8 @@ class TestVMLifeCycleSharedNwVPC(cloudstackTestCase): try: ssh = self.vm_1.get_ssh_client( - ipaddress=self.public_ip_1.ipaddress.ipaddress) + ipaddress=self.public_ip_1.ipaddress.ipaddress, + reconnect=True) self.debug("SSH into VM is successfully") except Exception as e: self.fail("Failed to SSH into instance") @@ -1569,7 +1586,8 @@ class TestVMLifeCycleSharedNwVPC(cloudstackTestCase): try: ssh = self.vm_1.get_ssh_client( - ipaddress=self.public_ip_1.ipaddress.ipaddress) + ipaddress=self.public_ip_1.ipaddress.ipaddress, + reconnect=True) self.debug("SSH into VM is successfully") except Exception as e: self.fail("Failed to SSH into instance") @@ -1860,8 +1878,11 @@ class TestVMLifeCycleBothIsolated(cloudstackTestCase): """Validating if the network rules (PF/LB) works properly or not?""" try: + self.debug("Checking if we can SSH into VM_1 through %s?" % + (self.public_ip_1.ipaddress.ipaddress)) ssh_1 = self.vm_1.get_ssh_client( - ipaddress=self.public_ip_1.ipaddress.ipaddress) + ipaddress=self.public_ip_1.ipaddress.ipaddress, + reconnect=True) self.debug("SSH into VM is successfully") self.debug("Verifying if we can ping to outside world from VM?") @@ -2285,8 +2306,11 @@ class TestVMLifeCycleStoppedVPCVR(cloudstackTestCase): def validate_network_rules(self): """Validates if the network rules work properly or not?""" try: + self.debug("Checking if we can SSH into VM_1 through %s?" % + (self.public_ip_1.ipaddress.ipaddress)) ssh_1 = self.vm_1.get_ssh_client( - ipaddress=self.public_ip_1.ipaddress.ipaddress) + ipaddress=self.public_ip_1.ipaddress.ipaddress, + reconnect=True) self.debug("SSH into VM is successfully") self.debug("Verifying if we can ping to outside world from VM?") @@ -2311,7 +2335,8 @@ class TestVMLifeCycleStoppedVPCVR(cloudstackTestCase): self.debug("Checking if we can SSH into VM_1?") try: ssh_2 = self.vm_1.get_ssh_client( - ipaddress=self.public_ip_2.ipaddress.ipaddress) + ipaddress=self.public_ip_2.ipaddress.ipaddress, + reconnect=True) self.debug("SSH into VM is successfully") self.debug("Verifying if we can ping to outside world from VM?") @@ -2595,7 +2620,8 @@ class TestVMLifeCycleStoppedVPCVR(cloudstackTestCase): try: ssh = self.vm_1.get_ssh_client( - ipaddress=self.public_ip_1.ipaddress.ipaddress) + ipaddress=self.public_ip_1.ipaddress.ipaddress, + reconnect=True) self.debug("SSH into VM is successfully") except Exception as e: self.fail("Failed to SSH into instance") @@ -2649,7 +2675,8 @@ class TestVMLifeCycleStoppedVPCVR(cloudstackTestCase): try: ssh = self.vm_1.get_ssh_client( - ipaddress=self.public_ip_1.ipaddress.ipaddress) + ipaddress=self.public_ip_1.ipaddress.ipaddress, + reconnect=True) self.debug("SSH into VM is successfully") except Exception as e: self.fail("Failed to SSH into instance") From 83692df9b2adc4a03adf0f8db16bc038e729f251 Mon Sep 17 00:00:00 2001 From: Rayees Namathponnan Date: Thu, 1 Aug 2013 17:26:27 -0700 Subject: [PATCH 076/763] Automation: fixed cleanup issue with test_vpn_user. Reviewed-by: Sheng Yang Signed-off-by: Sheng Yang --- test/integration/component/test_vpn_users.py | 53 +++++++++++--------- 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/test/integration/component/test_vpn_users.py b/test/integration/component/test_vpn_users.py index fe020d0f555..e327624cdad 100644 --- a/test/integration/component/test_vpn_users.py +++ b/test/integration/component/test_vpn_users.py @@ -19,6 +19,7 @@ """ # Import Local Modules from nose.plugins.attrib import attr +from marvin.cloudstackException import cloudstackAPIException from marvin.cloudstackTestCase import cloudstackTestCase from marvin.integration.lib.base import ( Account, @@ -127,32 +128,36 @@ class TestVPNUsers(cloudstackTestCase): return def setUp(self): - self.apiclient = self.testClient.getApiClient() - self.dbclient = self.testClient.getDbConnection() - self.account = Account.create( - self.apiclient, - self.services["account"], - domainid=self.domain.id - ) - self.virtual_machine = VirtualMachine.create( + try: + self.apiclient = self.testClient.getApiClient() + self.dbclient = self.testClient.getDbConnection() + self.account = Account.create( self.apiclient, - self.services["virtual_machine"], - templateid=self.template.id, - accountid=self.account.name, - domainid=self.account.domainid, - serviceofferingid=self.service_offering.id + self.services["account"], + domainid=self.domain.id ) - self.public_ip = PublicIPAddress.create( - self.apiclient, - self.virtual_machine.account, - self.virtual_machine.zoneid, - self.virtual_machine.domainid, - self.services["virtual_machine"] - ) - self.cleanup = [ - self.account, - ] - return + self.cleanup = [ + self.account, + ] + self.virtual_machine = VirtualMachine.create( + self.apiclient, + self.services["virtual_machine"], + templateid=self.template.id, + accountid=self.account.name, + domainid=self.account.domainid, + serviceofferingid=self.service_offering.id + ) + self.public_ip = PublicIPAddress.create( + self.apiclient, + self.virtual_machine.account, + self.virtual_machine.zoneid, + self.virtual_machine.domainid, + self.services["virtual_machine"] + ) + return + except cloudstackAPIException as e: + self.tearDown() + raise e def tearDown(self): try: From 2cbf97a6f1e4a6c942d6c2c0eb8a2f18c6eed67a Mon Sep 17 00:00:00 2001 From: Sheng Yang Date: Thu, 1 Aug 2013 18:47:40 -0700 Subject: [PATCH 077/763] Automation: More fix fox test_vpc_vm_life_cycle.py You cannot reach VPC router IP because you're in one of it's networks. As well as adding missing import for test_vpc_network_pfrules.py --- .../component/test_vpc_network_pfrules.py | 1 + .../component/test_vpc_vm_life_cycle.py | 169 ++++-------------- 2 files changed, 36 insertions(+), 134 deletions(-) diff --git a/test/integration/component/test_vpc_network_pfrules.py b/test/integration/component/test_vpc_network_pfrules.py index 274012da6a8..0d8e2f16e40 100644 --- a/test/integration/component/test_vpc_network_pfrules.py +++ b/test/integration/component/test_vpc_network_pfrules.py @@ -39,6 +39,7 @@ from marvin.integration.lib.common import (get_domain, cleanup_resources, list_routers) import socket +import time class Services: diff --git a/test/integration/component/test_vpc_vm_life_cycle.py b/test/integration/component/test_vpc_vm_life_cycle.py index 3531affadf2..9b10133efea 100644 --- a/test/integration/component/test_vpc_vm_life_cycle.py +++ b/test/integration/component/test_vpc_vm_life_cycle.py @@ -753,22 +753,10 @@ class TestVMLifeCycleVPC(cloudstackTestCase): except Exception as e: self.fail("Failed to SSH into instance") - # Find router associated with user account - routers = Router.list( - self.apiclient, - zoneid=self.zone.id, - listall=True - ) - self.assertEqual( - isinstance(routers, list), - True, - "Check list response returns a valid list" - ) - router = routers[0] self.debug("check the userdata with that of present in router") try: cmds = [ - "wget http://%s/latest/user-data" % router.guestipaddress, + "wget http://%s/latest/user-data" % self.network_1.gateway, "cat user-data", ] for c in cmds: @@ -808,23 +796,11 @@ class TestVMLifeCycleVPC(cloudstackTestCase): except Exception as e: self.fail("Failed to SSH into instance") - # Find router associated with user account - routers = Router.list( - self.apiclient, - zoneid=self.zone.id, - listall=True - ) - self.assertEqual( - isinstance(routers, list), - True, - "Check list response returns a valid list" - ) - router = routers[0] self.debug("check the metadata with that of present in router") try: cmds = [ - "wget http://%s/latest/meta-data" % router.guestipaddress, - "cat user-data", + "wget http://%s/latest/vm-id" % self.network_1.gateway, + "cat vm-id", ] for c in cmds: result = ssh.execute(c) @@ -872,28 +848,19 @@ class TestVMLifeCycleVPC(cloudstackTestCase): ) # Check if the network rules still exists after Vm stop - self.debug("Checking if NAT rules ") - nat_rules = NATRule.list( - self.apiclient, - id=self.nat_rule.id, - listall=True - ) - self.assertEqual( - nat_rules, - None, - "List NAT rules should not return anything" - ) + self.debug("Checking if NAT rules existed") + with self.assertRaises(Exception): + nat_rules = NATRule.list( + self.apiclient, + id=self.nat_rule.id, + listall=True + ) - lb_rules = LoadBalancerRule.list( + lb_rules = LoadBalancerRule.list( self.apiclient, id=self.lb_rule.id, listall=True ) - self.assertEqual( - lb_rules, - None, - "List LB rules should not return anything" - ) return class TestVMLifeCycleSharedNwVPC(cloudstackTestCase): @@ -1537,22 +1504,10 @@ class TestVMLifeCycleSharedNwVPC(cloudstackTestCase): except Exception as e: self.fail("Failed to SSH into instance") - # Find router associated with user account - routers = Router.list( - self.apiclient, - zoneid=self.zone.id, - listall=True - ) - self.assertEqual( - isinstance(routers, list), - True, - "Check list response returns a valid list" - ) - router = routers[0] self.debug("check the userdata with that of present in router") try: cmds = [ - "wget http://%s/latest/user-data" % router.guestipaddress, + "wget http://%s/latest/user-data" % self.network_1.gateway, "cat user-data", ] for c in cmds: @@ -1592,23 +1547,11 @@ class TestVMLifeCycleSharedNwVPC(cloudstackTestCase): except Exception as e: self.fail("Failed to SSH into instance") - # Find router associated with user account - routers = Router.list( - self.apiclient, - zoneid=self.zone.id, - listall=True - ) - self.assertEqual( - isinstance(routers, list), - True, - "Check list response returns a valid list" - ) - router = routers[0] self.debug("check the metadata with that of present in router") try: cmds = [ - "wget http://%s/latest/meta-data" % router.guestipaddress, - "cat user-data", + "wget http://%s/latest/vm-id" % self.network_1.gateway, + "cat vm-id", ] for c in cmds: result = ssh.execute(c) @@ -1674,29 +1617,20 @@ class TestVMLifeCycleSharedNwVPC(cloudstackTestCase): ["expunge.interval", "expunge.delay"] ) - # Check if the network rules still exists after Vm stop - self.debug("Checking if NAT rules ") - nat_rules = NATRule.list( - self.apiclient, - id=self.nat_rule.id, - listall=True - ) - self.assertEqual( - nat_rules, - None, - "List NAT rules should not return anything" - ) + # Check if the network rules still exists after Vm expunged + self.debug("Checking if NAT rules existed ") + with self.assertRaises(Exception): + nat_rules = NATRule.list( + self.apiclient, + id=self.nat_rule.id, + listall=True + ) - lb_rules = LoadBalancerRule.list( + lb_rules = LoadBalancerRule.list( self.apiclient, id=self.lb_rule.id, listall=True ) - self.assertEqual( - lb_rules, - None, - "List LB rules should not return anything" - ) return @@ -2626,22 +2560,10 @@ class TestVMLifeCycleStoppedVPCVR(cloudstackTestCase): except Exception as e: self.fail("Failed to SSH into instance") - # Find router associated with user account - routers = Router.list( - self.apiclient, - zoneid=self.zone.id, - listall=True - ) - self.assertEqual( - isinstance(routers, list), - True, - "Check list response returns a valid list" - ) - router = routers[0] self.debug("check the userdata with that of present in router") try: cmds = [ - "wget http://%s/latest/user-data" % router.guestipaddress, + "wget http://%s/latest/user-data" % self.network_1.gateway, "cat user-data", ] for c in cmds: @@ -2681,23 +2603,11 @@ class TestVMLifeCycleStoppedVPCVR(cloudstackTestCase): except Exception as e: self.fail("Failed to SSH into instance") - # Find router associated with user account - routers = Router.list( - self.apiclient, - zoneid=self.zone.id, - listall=True - ) - self.assertEqual( - isinstance(routers, list), - True, - "Check list response returns a valid list" - ) - router = routers[0] self.debug("check the metadata with that of present in router") try: cmds = [ - "wget http://%s/latest/meta-data" % router.guestipaddress, - "cat user-data", + "wget http://%s/latest/vm-id" % self.network_1.gateway, + "cat vm-id", ] for c in cmds: result = ssh.execute(c) @@ -2744,27 +2654,18 @@ class TestVMLifeCycleStoppedVPCVR(cloudstackTestCase): ["expunge.interval", "expunge.delay"] ) - # Check if the network rules still exists after Vm stop - self.debug("Checking if NAT rules ") - nat_rules = NATRule.list( - self.apiclient, - id=self.nat_rule.id, - listall=True - ) - self.assertEqual( - nat_rules, - None, - "List NAT rules should not return anything" - ) + # Check if the network rules still exists after Vm expunged + self.debug("Checking if NAT rules existed ") + with self.assertRaises(Exception): + nat_rules = NATRule.list( + self.apiclient, + id=self.nat_rule.id, + listall=True + ) - lb_rules = LoadBalancerRule.list( + lb_rules = LoadBalancerRule.list( self.apiclient, id=self.lb_rule.id, listall=True ) - self.assertEqual( - lb_rules, - None, - "List LB rules should not return anything" - ) return From 7eb20b8500a002ad75f73dc01bd67dd14da6bdcf Mon Sep 17 00:00:00 2001 From: Kishan Kavala Date: Fri, 2 Aug 2013 11:58:48 +0530 Subject: [PATCH 078/763] CLOUDSTACK-4032: Added KVM.snapshot.enabled config to 41to42 upgrade --- server/src/com/cloud/configuration/Config.java | 4 +++- setup/db/db/schema-410to420.sql | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/server/src/com/cloud/configuration/Config.java b/server/src/com/cloud/configuration/Config.java index 0d7269414e7..7bc7193aa03 100755 --- a/server/src/com/cloud/configuration/Config.java +++ b/server/src/com/cloud/configuration/Config.java @@ -572,7 +572,9 @@ public enum Config { return "TemplateManager"; } else if (_componentClass == VpcManager.class) { return "VpcManager"; - }else { + } else if (_componentClass == SnapshotManager.class) { + return "SnapshotManager"; + } else { return "none"; } } diff --git a/setup/db/db/schema-410to420.sql b/setup/db/db/schema-410to420.sql index 3f25c3b7e59..de4b7658461 100644 --- a/setup/db/db/schema-410to420.sql +++ b/setup/db/db/schema-410to420.sql @@ -2221,4 +2221,6 @@ CREATE VIEW `cloud`.`account_vmstats_view` AS `cloud`.`vm_instance` where vm_type = 'User' - group by account_id , state; \ No newline at end of file + group by account_id , state; + +INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Snapshots', 'DEFAULT', 'SnapshotManager', 'KVM.snapshot.enabled', 'false', 'whether snapshot is enabled for KVM hosts'); \ No newline at end of file From de965128775f7dbe393140c14a9d4eab30f085ef Mon Sep 17 00:00:00 2001 From: Prasanna Santhanam Date: Fri, 2 Aug 2013 12:51:45 +0530 Subject: [PATCH 079/763] Remove duplicate tests for stopped_vm Signed-off-by: Prasanna Santhanam --- test/integration/component/test_stopped_vm.py | 142 ------------------ 1 file changed, 142 deletions(-) diff --git a/test/integration/component/test_stopped_vm.py b/test/integration/component/test_stopped_vm.py index e1f5f2eb2f8..c845f608c34 100644 --- a/test/integration/component/test_stopped_vm.py +++ b/test/integration/component/test_stopped_vm.py @@ -1324,148 +1324,6 @@ class TestDeployVMBasicZone(cloudstackTestCase): except Exception as e: self.debug("Warning! Exception in tearDown: %s" % e) - @attr(tags = ["eip", "basic", "sg"]) - def test_01_deploy_vm_startvm_true(self): - """Test Deploy Virtual Machine with startVM=true parameter - """ - - # Validate the following: - # 1. deploy Vm with the startvm=true - # 2. Should be able to login to the VM. - # 3. listVM command should return the deployed VM.State of this VM - # should be "Running". - - self.debug("Checking the network type of the zone: %s" % - self.zone.networktype) - self.assertEqual( - self.zone.networktype, - 'Basic', - "Zone must be configured in basic networking mode" - ) - self.debug("Deploying instance in the account: %s" % - self.account.name) - self.virtual_machine = VirtualMachine.create( - self.apiclient, - self.services["virtual_machine"], - accountid=self.account.name, - domainid=self.account.domainid, - serviceofferingid=self.service_offering.id, - startvm=True, - diskofferingid=self.disk_offering.id, - mode=self.zone.networktype - ) - - self.debug("Deployed instance ion account: %s" % - self.account.name) - list_vm_response = list_virtual_machines( - self.apiclient, - id=self.virtual_machine.id - ) - - self.debug( - "Verify listVirtualMachines response for virtual machine: %s" \ - % self.virtual_machine.id - ) - - self.assertEqual( - isinstance(list_vm_response, list), - True, - "Check list response returns a valid list" - ) - vm_response = list_vm_response[0] - - self.assertEqual( - - vm_response.state, - "Running", - "VM should be in Running state after deployment" - ) - return - - @attr(tags = ["eip", "basic", "sg"]) - def test_02_deploy_vm_startvm_false(self): - """Test Deploy Virtual Machine with startVM=true parameter - """ - - # Validate the following: - # 1. deploy Vm with the startvm=true - # 2. Should be able to login to the VM. - # 3. listVM command should return the deployed VM.State of this VM - # should be "Running". - - self.debug("Checking the network type of the zone: %s" % - self.zone.networktype) - self.assertEqual( - self.zone.networktype, - 'Basic', - "Zone must be configured in basic networking mode" - ) - self.debug("Deploying instance in the account: %s" % - self.account.name) - self.virtual_machine = VirtualMachine.create( - self.apiclient, - self.services["virtual_machine"], - accountid=self.account.name, - domainid=self.account.domainid, - serviceofferingid=self.service_offering.id, - startvm=False, - mode=self.zone.networktype - ) - - self.debug("Deployed instance in account: %s" % - self.account.name) - list_vm_response = list_virtual_machines( - self.apiclient, - id=self.virtual_machine.id - ) - - self.debug( - "Verify listVirtualMachines response for virtual machine: %s" \ - % self.virtual_machine.id - ) - - self.assertEqual( - isinstance(list_vm_response, list), - True, - "Check list response returns a valid list" - ) - vm_response = list_vm_response[0] - - self.assertEqual( - - vm_response.state, - "Stopped", - "VM should be in stopped state after deployment" - ) - self.debug("Starting the instance: %s" % self.virtual_machine.name) - self.virtual_machine.start(self.apiclient) - self.debug("Started the instance: %s" % self.virtual_machine.name) - - list_vm_response = list_virtual_machines( - self.apiclient, - id=self.virtual_machine.id - ) - - self.debug( - "Verify listVirtualMachines response for virtual machine: %s" \ - % self.virtual_machine.id - ) - - self.assertEqual( - isinstance(list_vm_response, list), - True, - "Check list response returns a valid list" - ) - vm_response = list_vm_response[0] - - self.assertEqual( - - vm_response.state, - "Running", - "VM should be in running state after deployment" - ) - return - class TestDeployVMFromTemplate(cloudstackTestCase): From 5e72ddec85a5d018fe087d4f8dc77bc4ccc91db2 Mon Sep 17 00:00:00 2001 From: Devdeep Singh Date: Fri, 2 Aug 2013 13:16:01 +0530 Subject: [PATCH 080/763] CLOUDSTACK-4009: healthcheck.update.interval global config missing after upgrade. Added the parameter in the db schema file. --- setup/db/db/schema-410to420.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup/db/db/schema-410to420.sql b/setup/db/db/schema-410to420.sql index de4b7658461..6fdf3fea267 100644 --- a/setup/db/db/schema-410to420.sql +++ b/setup/db/db/schema-410to420.sql @@ -2223,4 +2223,5 @@ CREATE VIEW `cloud`.`account_vmstats_view` AS vm_type = 'User' group by account_id , state; -INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Snapshots', 'DEFAULT', 'SnapshotManager', 'KVM.snapshot.enabled', 'false', 'whether snapshot is enabled for KVM hosts'); \ No newline at end of file +INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Advanced', 'DEFAULT', 'management-server', 'healthcheck.update.interval', '600', 'Time Interval to fetch the LB health check states (in sec)'); +INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Snapshots', 'DEFAULT', 'SnapshotManager', 'KVM.snapshot.enabled', 'false', 'whether snapshot is enabled for KVM hosts'); From 725d281cb28c89b8f1536de769d7acf87f53fbe6 Mon Sep 17 00:00:00 2001 From: JijunLiu Date: Sun, 14 Jul 2013 11:21:39 +0800 Subject: [PATCH 081/763] Add cpu model for kvm guest.Now all the kvm guest's cpu model is 'QEMU Virtual CPU version xxx'. This will affect the activation of Windows OS and low performance. I add three mode for user to indicate the guest cpu model.add libvirt version check Signed-off-by: Wei Zhou (cherry picked from commit 2903bb5fd9d59803980102854d11e33ebe56088b) --- agent/conf/agent.properties | 20 ++++++++++++++ .../resource/LibvirtComputingResource.java | 27 +++++++++++++++++++ .../hypervisor/kvm/resource/LibvirtVMDef.java | 27 +++++++++++++++++++ .../kvm/resource/LibvirtVMDefTest.java | 20 ++++++++++++++ 4 files changed, 94 insertions(+) diff --git a/agent/conf/agent.properties b/agent/conf/agent.properties index 60030ae4f11..5f5f3682afd 100644 --- a/agent/conf/agent.properties +++ b/agent/conf/agent.properties @@ -94,3 +94,23 @@ domr.scripts.dir=scripts/network/domr/kvm # libvirt.vif.driver=com.cloud.hypervisor.kvm.resource.DirectVifDriver # network.direct.source.mode=private # network.direct.device=eth0 + +# setting to enable the cpu model to kvm guest globally. +# three option:custom,host-model and host-passthrough. +# custom - user custom the CPU model which specified by guest.cpu.model. +# host-model - identify the named CPU model which most closely matches the host, +# and then request additional CPU flags to complete the match. This should give +# close to maximum functionality/performance, which maintaining good +# reliability/compatibility if the guest is migrated to another host with slightly different host CPUs. +# host-passthrough - tell KVM to passthrough the host CPU with no modifications. +# The difference to host-model, instead of just matching feature flags, +# every last detail of the host CPU is matched. This gives absolutely best performance, +# and can be important to some apps which check low level CPU details, +# but it comes at a cost wrt migration. The guest can only be migrated to +# an exactly matching host CPU. +# +# guest.cpu.mode=custom|host-model|host-passthrough +# This param is only valid if guest.cpu.mode=custom, +# for examples:"Conroe" "Penryn", "Nehalem", "Westmere", "pentiumpro" and so +# on,run virsh capabilities for more details. +# guest.cpu.model= diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 7330fa7daf4..2ae21addec8 100755 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -192,6 +192,7 @@ import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.hypervisor.kvm.resource.KVMHABase.NfsStoragePool; import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.ClockDef; import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.ConsoleDef; +import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.CpuModeDef; import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.CpuTuneDef; import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.DevicesDef; import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.DiskDef; @@ -369,6 +370,8 @@ ServerResource { private boolean _can_bridge_firewall; protected String _localStoragePath; protected String _localStorageUUID; + protected String _guestCpuMode; + protected String _guestCpuModel; private final Map _pifs = new HashMap(); private final Map _vmStats = new ConcurrentHashMap(); @@ -760,6 +763,18 @@ ServerResource { s_logger.trace("Ignoring libvirt error.", e); } + _guestCpuMode = (String) params.get("guest.cpu.mode"); + if (_guestCpuMode != null) { + _guestCpuModel = (String) params.get("guest.cpu.model"); + + if(_hypervisorLibvirtVersion < (9 * 1000 + 10)) { + s_logger.warn("LibVirt version 0.9.10 required for guest cpu mode, but version " + + prettyVersion(_hypervisorLibvirtVersion) + " detected, so it will be disabled"); + _guestCpuMode = null; + _guestCpuModel = null; + } + } + String[] info = NetUtils.getNetworkParams(_privateNic); _monitor = new KVMHAMonitor(null, info[0], _heartBeatPath); @@ -3289,6 +3304,11 @@ ServerResource { grd.setVcpuNum(vmTO.getCpus()); vm.addComp(grd); + CpuModeDef cmd = new CpuModeDef(); + cmd.setMode(_guestCpuMode); + cmd.setModel(_guestCpuModel); + vm.addComp(cmd); + CpuTuneDef ctd = new CpuTuneDef(); /** A 4.0.X/4.1.X management server doesn't send the correct JSON @@ -4953,6 +4973,13 @@ ServerResource { return new Answer(cmd, success, ""); } + private String prettyVersion(long version) { + long major = version / 1000000; + long minor = version % 1000000 / 1000; + long release = version % 1000000 % 1000; + return major + "." + minor + "." + release; + } + @Override public void setName(String name) { // TODO Auto-generated method stub diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java index 51208702169..9a3bef9f381 100644 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java @@ -938,6 +938,33 @@ public class LibvirtVMDef { } } + public static class CpuModeDef { + private String _mode; + private String _model; + + public void setMode(String mode) { + _mode = mode; + } + + public void setModel(String model) { + _model = model; + } + + @Override + public String toString() { + StringBuilder modeBuidler = new StringBuilder(); + if ("custom".equalsIgnoreCase(_mode) && _model != null) { + modeBuidler.append("" + + _model + ""); + } else if ("host-model".equals(_mode)) { + modeBuidler.append(""); + } else if ("host-passthrough".equals(_mode)) { + modeBuidler.append(""); + } + return modeBuidler.toString(); + } + } + public static class SerialDef { private final String _type; private final String _source; diff --git a/plugins/hypervisors/kvm/test/com/cloud/hypervisor/kvm/resource/LibvirtVMDefTest.java b/plugins/hypervisors/kvm/test/com/cloud/hypervisor/kvm/resource/LibvirtVMDefTest.java index 2c0ff8d8b77..9db2902fe25 100644 --- a/plugins/hypervisors/kvm/test/com/cloud/hypervisor/kvm/resource/LibvirtVMDefTest.java +++ b/plugins/hypervisors/kvm/test/com/cloud/hypervisor/kvm/resource/LibvirtVMDefTest.java @@ -49,4 +49,24 @@ public class LibvirtVMDefTest extends TestCase { assertEquals(expected, ifDef.toString()); } + public void testCpuModeDef(){ + LibvirtVMDef.CpuModeDef cpuModeDef = new LibvirtVMDef.CpuModeDef(); + cpuModeDef.setMode("custom"); + cpuModeDef.setModel("Nehalem"); + + String expected1 = "Nehalem"; + + assertEquals(expected1, cpuModeDef.toString()); + + cpuModeDef.setMode("host-model"); + String expected2 = ""; + + assertEquals(expected2, cpuModeDef.toString()); + + cpuModeDef.setMode("host-passthrough"); + String expected3 = ""; + assertEquals(expected3, cpuModeDef.toString()); + + } + } From fd9cf9743b8bdbf446a8db1d735984d97b14f452 Mon Sep 17 00:00:00 2001 From: Gaurav Aradhye Date: Wed, 31 Jul 2013 01:14:05 -0400 Subject: [PATCH 082/763] CLOUDSTACK:3594: Fixes related to affinity groups tests. Included two test suites those were missed because there was no decorator for test cases Signed-off-by: Prasanna Santhanam --- .../component/test_affinity_groups.py | 244 ++++++++++-------- tools/marvin/marvin/integration/lib/base.py | 14 + 2 files changed, 151 insertions(+), 107 deletions(-) diff --git a/test/integration/component/test_affinity_groups.py b/test/integration/component/test_affinity_groups.py index 3ecab6ff66d..309adc6a60f 100644 --- a/test/integration/component/test_affinity_groups.py +++ b/test/integration/component/test_affinity_groups.py @@ -51,12 +51,8 @@ class Services: # In MBs }, "ostype": 'CentOS 5.3 (64-bit)', - "host_anti_affinity_0": { - "name": "TestAffGrp_HA_0", - "type": "host anti-affinity", - }, - "host_anti_affinity_1": { - "name": "TestAffGrp_HA_1", + "host_anti_affinity": { + "name": "", "type": "host anti-affinity", }, "virtual_machine" : { @@ -151,17 +147,22 @@ class TestCreateAffinityGroup(cloudstackTestCase): raise Exception("Warning: Exception during cleanup : %s" % e) def create_aff_grp(self, api_client=None, aff_grp=None, - acc=None, domainid=None): + acc=None, domainid=None, aff_grp_name=None): if not api_client: api_client = self.api_client if not aff_grp: - self.services["host_anti_affinity_0"] + aff_grp = self.services["host_anti_affinity"] if not acc: acc = self.account.name if not domainid: domainid = self.domain.id + if aff_grp_name is None: + aff_grp["name"] = "aff_grp_" + random_gen(size=6) + else: + aff_grp["name"] = aff_grp_name + try: return AffinityGroup.create(api_client, aff_grp, acc, domainid) except Exception as e: @@ -174,7 +175,7 @@ class TestCreateAffinityGroup(cloudstackTestCase): @return: """ - aff_grp = self.create_aff_grp(aff_grp=self.services["host_anti_affinity_0"], + aff_grp = self.create_aff_grp(aff_grp=self.services["host_anti_affinity"], acc=self.account.name, domainid=self.account.domainid) self.debug("Created Affinity Group: %s" % aff_grp.name) list_aff_grps = AffinityGroup.list(self.api_client, id=aff_grp.id) @@ -197,7 +198,7 @@ class TestCreateAffinityGroup(cloudstackTestCase): domainapiclient = self.testClient.createUserApiClient(self.do_admin.name, self.new_domain.name, 2) - aff_grp = self.create_aff_grp(api_client=domainapiclient, aff_grp=self.services["host_anti_affinity_0"], + aff_grp = self.create_aff_grp(api_client=domainapiclient, aff_grp=self.services["host_anti_affinity"], acc=self.do_admin.name, domainid=self.new_domain.id) aff_grp.delete(domainapiclient) @@ -214,7 +215,7 @@ class TestCreateAffinityGroup(cloudstackTestCase): userapiclient = self.testClient.createUserApiClient(self.user.name, self.domain.name) self.cleanup.append(self.user) - aff_grp = self.create_aff_grp(api_client=userapiclient, aff_grp=self.services["host_anti_affinity_0"], + aff_grp = self.create_aff_grp(api_client=userapiclient, aff_grp=self.services["host_anti_affinity"], acc=self.user.name, domainid=self.domain.id) aff_grp.delete(userapiclient) @@ -230,11 +231,12 @@ class TestCreateAffinityGroup(cloudstackTestCase): domainid=self.domain.id) self.cleanup.append(self.user) - aff_grp = self.create_aff_grp(aff_grp=self.services["host_anti_affinity_0"], + aff_grp = self.create_aff_grp(aff_grp=self.services["host_anti_affinity"], acc=self.user.name, domainid=self.domain.id) with self.assertRaises(Exception): - self.create_aff_grp(aff_grp=self.services["host_anti_affinity_0"], - acc=self.user.name, domainid=self.domain.id) + self.create_aff_grp(aff_grp=self.services["host_anti_affinity"], + acc=self.user.name, domainid=self.domain.id, + aff_grp_name = aff_grp.name) self.debug("Deleted Affinity Group: %s" %aff_grp.name) aff_grp.delete(self.api_client) @@ -250,11 +252,11 @@ class TestCreateAffinityGroup(cloudstackTestCase): domainid=self.domain.id) self.cleanup.append(self.user) - aff_grp = self.create_aff_grp(aff_grp=self.services["host_anti_affinity_0"], + aff_grp = self.create_aff_grp(aff_grp=self.services["host_anti_affinity"], acc=self.user.name, domainid=self.domain.id) try: - self.create_aff_grp(aff_grp=self.services["host_anti_affinity_0"]) + self.create_aff_grp(aff_grp=self.services["host_anti_affinity"]) except Exception: self.debug("Error: Creating affinity group with same name from different account failed.") @@ -345,11 +347,15 @@ class TestListAffinityGroups(cloudstackTestCase): if api_client == None: api_client = self.api_client if aff_grp == None: - self.services["host_anti_affinity_0"] + aff_grp = self.services["host_anti_affinity"] + + aff_grp["name"] = "aff_grp_" + random_gen(size=6) try: - self.aff_grp.append(AffinityGroup.create(api_client, - aff_grp, acc, domainid)) + aff_grp = AffinityGroup.create(api_client, + aff_grp, acc, domainid) + self.aff_grp.append(aff_grp) + return aff_grp except Exception as e: raise Exception("Error: Creation of Affinity Group failed : %s" %e) @@ -375,12 +381,13 @@ class TestListAffinityGroups(cloudstackTestCase): msg="VM is not in Running state") return vm, vm_response.hostid + @attr(tags=["simulator", "basic", "advanced"]) def test_01_list_aff_grps_for_vm(self): """ List affinity group for a vm """ - self.create_aff_grp(aff_grp=self.services["host_anti_affinity_0"]) + self.create_aff_grp(aff_grp=self.services["host_anti_affinity"]) list_aff_grps = AffinityGroup.list(self.api_client) vm, hostid = self.create_vm_in_aff_grps([self.aff_grp[0].name]) @@ -394,15 +401,16 @@ class TestListAffinityGroups(cloudstackTestCase): #Wait for expunge interval to cleanup VM wait_for_cleanup(self.apiclient, ["expunge.delay", "expunge.interval"]) - AffinityGroup.delete(self.api_client, self.aff_grp[0].name) + self.aff_grp[0].delete(self.api_client) + @attr(tags=["simulator", "basic", "advanced"]) def test_02_list_multiple_aff_grps_for_vm(self): """ List multiple affinity groups associated with a vm """ - self.create_aff_grp(aff_grp=self.services["host_anti_affinity_0"]) - self.create_aff_grp(aff_grp=self.services["host_anti_affinity_1"]) + aff_grp_01 = self.create_aff_grp(aff_grp=self.services["host_anti_affinity"]) + aff_grp_02 = self.create_aff_grp(aff_grp=self.services["host_anti_affinity"]) aff_grps_names = [self.aff_grp[0].name, self.aff_grp[1].name] vm, hostid = self.create_vm_in_aff_grps(aff_grps_names) @@ -421,60 +429,65 @@ class TestListAffinityGroups(cloudstackTestCase): #Wait for expunge interval to cleanup VM wait_for_cleanup(self.apiclient, ["expunge.delay", "expunge.interval"]) - [AffinityGroup.delete(self.api_client, name) for name in aff_grps_names] + aff_grp_01.delete(self.api_client) + aff_grp_02.delete(self.api_client) + @attr(tags=["simulator", "basic", "advanced"]) def test_03_list_aff_grps_by_id(self): """ List affinity groups by id """ - self.create_aff_grp(aff_grp=self.services["host_anti_affinity_0"]) + self.create_aff_grp(aff_grp=self.services["host_anti_affinity"]) print self.aff_grp[0].__dict__ list_aff_grps = AffinityGroup.list(self.api_client) list_aff_grps = AffinityGroup.list(self.api_client, id=list_aff_grps[0].id) self.assertEqual(list_aff_grps[0].name, self.aff_grp[0].name, "Listing Affinity Group by VM id failed") - AffinityGroup.delete(self.api_client, self.aff_grp[0].name) + self.aff_grp[0].delete(self.api_client) + @attr(tags=["simulator", "basic", "advanced"]) def test_04_list_aff_grps_by_name(self): """ List Affinity Groups by name """ - self.create_aff_grp(aff_grp=self.services["host_anti_affinity_0"]) + self.create_aff_grp(aff_grp=self.services["host_anti_affinity"]) list_aff_grps = AffinityGroup.list(self.api_client, name=self.aff_grp[0].name) self.assertEqual(list_aff_grps[0].name, self.aff_grp[0].name, "Listing Affinity Group by name failed") - AffinityGroup.delete(self.api_client, self.aff_grp[0].name) + self.aff_grp[0].delete(self.api_client) + @attr(tags=["simulator", "basic", "advanced"]) def test_05_list_aff_grps_by_non_existing_id(self): """ List Affinity Groups by non-existing id """ - self.create_aff_grp(aff_grp=self.services["host_anti_affinity_0"]) + self.create_aff_grp(aff_grp=self.services["host_anti_affinity"]) list_aff_grps = AffinityGroup.list(self.api_client, id=1234) self.assertEqual(list_aff_grps, None, "Listing Affinity Group by non-existing id succeeded.") - AffinityGroup.delete(self.api_client, self.aff_grp[0].name) + self.aff_grp[0].delete(self.api_client) + @attr(tags=["simulator", "basic", "advanced"]) def test_06_list_aff_grps_by_non_existing_name(self): """ List Affinity Groups by non-existing name """ - self.create_aff_grp(aff_grp=self.services["host_anti_affinity_0"]) + self.create_aff_grp(aff_grp=self.services["host_anti_affinity"]) list_aff_grps = AffinityGroup.list(self.api_client, name="NonexistingName") self.assertEqual(list_aff_grps, None, "Listing Affinity Group by non-existing name succeeded.") - AffinityGroup.delete(self.api_client, self.aff_grp[0].name) + self.aff_grp[0].delete(self.api_client) class TestDeleteAffinityGroups(cloudstackTestCase): @@ -547,7 +560,10 @@ class TestDeleteAffinityGroups(cloudstackTestCase): if api_client == None: api_client = self.api_client if aff_grp == None: - self.services["host_anti_affinity_0"] + aff_grp = self.services["host_anti_affinity"] + + aff_grp["name"] = "aff_grp_" + random_gen(size=6) + try: return AffinityGroup.create(api_client, aff_grp, acc, domainid) except Exception as e: @@ -589,7 +605,7 @@ class TestDeleteAffinityGroups(cloudstackTestCase): Delete Affinity Group by name """ - aff_0 = self.create_aff_grp(aff_grp=self.services["host_anti_affinity_0"]) + aff_0 = self.create_aff_grp(aff_grp=self.services["host_anti_affinity"]) AffinityGroup.list(self.api_client, name=aff_0.name) self.delete_aff_group(self.api_client, name=aff_0.name) self.assert_(AffinityGroup.list(self.api_client, name=aff_0.name) is None) @@ -600,9 +616,9 @@ class TestDeleteAffinityGroups(cloudstackTestCase): Delete Affinity Group as admin for an account """ - aff_0 = self.create_aff_grp(aff_grp=self.services["host_anti_affinity_0"], + aff_0 = self.create_aff_grp(aff_grp=self.services["host_anti_affinity"], acc=self.account.name, domainid=self.domain.id) - aff_1 = self.create_aff_grp(aff_grp=self.services["host_anti_affinity_1"], + aff_1 = self.create_aff_grp(aff_grp=self.services["host_anti_affinity"], acc=self.account.name, domainid=self.domain.id) aff_0.delete(self.api_client) @@ -616,14 +632,12 @@ class TestDeleteAffinityGroups(cloudstackTestCase): Delete Affinity Group which has vms in it """ - aff_0 = self.create_aff_grp(aff_grp=self.services["host_anti_affinity_0"], - acc=self.account.name, domainid=self.domain.id) - aff_1 = self.create_aff_grp(aff_grp=self.services["host_anti_affinity_1"], - acc=self.account.name, domainid=self.domain.id) + aff_0 = self.create_aff_grp(aff_grp=self.services["host_anti_affinity"]) + aff_1 = self.create_aff_grp(aff_grp=self.services["host_anti_affinity"]) vm, hostid = self.create_vm_in_aff_grps([aff_0.name, aff_1.name]) aff_0.delete(self.api_client) - vm_list = list_virtual_machines(self.apiclient, id=self.virtual_machine.id) + vm_list = list_virtual_machines(self.apiclient, id=vm.id) self.assert_(vm_list is not None) vm.delete(self.api_client) #Wait for expunge interval to cleanup VM @@ -636,7 +650,7 @@ class TestDeleteAffinityGroups(cloudstackTestCase): Delete Affinity Group which has vms after updating affinity group for vms in it """ - aff_0 = self.create_aff_grp(aff_grp=self.services["host_anti_affinity_0"]) + aff_0 = self.create_aff_grp(aff_grp=self.services["host_anti_affinity"]) vm1, hostid1 = self.create_vm_in_aff_grps([aff_0.name]) vm2, hostid2 = self.create_vm_in_aff_grps([aff_0.name]) @@ -664,7 +678,7 @@ class TestDeleteAffinityGroups(cloudstackTestCase): self.services["new_account"]) self.cleanup.append(self.user1) - aff_0 = self.create_aff_grp(aff_grp=self.services["host_anti_affinity_0"], + aff_0 = self.create_aff_grp(aff_grp=self.services["host_anti_affinity"], acc=self.user1.name, domainid=self.domain.id) @@ -677,7 +691,7 @@ class TestDeleteAffinityGroups(cloudstackTestCase): acctType=0) aff_1 = self.create_aff_grp(api_client=userapiclient, - aff_grp=self.services["host_anti_affinity_1"]) + aff_grp=self.services["host_anti_affinity"]) list_aff_grps = AffinityGroup.list(self.api_client, name=aff_0.name) @@ -700,7 +714,7 @@ class TestDeleteAffinityGroups(cloudstackTestCase): self.services["new_account"]) self.cleanup.append(self.user1) - aff_0 = self.create_aff_grp(aff_grp=self.services["host_anti_affinity_0"], + aff_0 = self.create_aff_grp(aff_grp=self.services["host_anti_affinity"], acc=self.user1.name, domainid=self.domain.id) @@ -713,7 +727,7 @@ class TestDeleteAffinityGroups(cloudstackTestCase): acctType=0) aff_1 = self.create_aff_grp(api_client=userapiclient, - aff_grp=self.services["host_anti_affinity_1"]) + aff_grp=self.services["host_anti_affinity"]) list_aff_grps = AffinityGroup.list(self.api_client, name=aff_0.name) @@ -797,7 +811,10 @@ class TestUpdateVMAffinityGroups(cloudstackTestCase): if api_client == None: api_client = self.api_client if aff_grp == None: - self.services["host_anti_affinity_0"] + aff_grp = self.services["host_anti_affinity"] + + aff_grp["name"] = "aff_grp_" + random_gen(size=6) + try: self.aff_grp.append(AffinityGroup.create(api_client, aff_grp, acc, domainid)) @@ -805,16 +822,18 @@ class TestUpdateVMAffinityGroups(cloudstackTestCase): raise Exception("Error: Creation of Affinity Group failed : %s" %e) def create_vm_in_aff_grps(self, ag_list): - self.debug('Creating VM in AffinityGroup=%s' % ag_list[0]) + + self.debug('Creating VM in AffinityGroup=%s' % ag_list) + vm = VirtualMachine.create( - self.api_client, + self.api_client, self.services["virtual_machine"], templateid=self.template.id, serviceofferingid=self.service_offering.id, affinitygroupnames=ag_list ) self.debug('Created VM=%s in Affinity Group=%s' % - (vm.id, ag_list[0])) + (vm.id, ag_list)) list_vm = list_virtual_machines(self.api_client, id=vm.id) @@ -829,13 +848,14 @@ class TestUpdateVMAffinityGroups(cloudstackTestCase): return vm, vm_response.hostid + @attr(tags=["simulator", "basic", "advanced", "multihost"]) def test_01_update_aff_grp_by_ids(self): """ Update the list of affinityGroups by using affinity groupids """ - self.create_aff_grp(aff_grp=self.services["host_anti_affinity_0"]) - self.create_aff_grp(aff_grp=self.services["host_anti_affinity_1"]) + self.create_aff_grp(aff_grp=self.services["host_anti_affinity"]) + self.create_aff_grp(aff_grp=self.services["host_anti_affinity"]) vm1, hostid1 = self.create_vm_in_aff_grps([self.aff_grp[0].name]) vm2, hostid2 = self.create_vm_in_aff_grps([self.aff_grp[0].name]) @@ -873,16 +893,17 @@ class TestUpdateVMAffinityGroups(cloudstackTestCase): vm2.delete(self.api_client) #Wait for expunge interval to cleanup VM wait_for_cleanup(self.apiclient, ["expunge.delay", "expunge.interval"]) - for i in aff_grps_names: - AffinityGroup.delete(self.api_client, i) + for aff_grp in self.aff_grp: + aff_grp.delete(self.api_client) + @attr(tags=["simulator", "basic", "advanced", "multihost"]) def test_02_update_aff_grp_by_names(self): """ Update the list of affinityGroups by using affinity groupnames """ - self.create_aff_grp(aff_grp=self.services["host_anti_affinity_0"]) - self.create_aff_grp(aff_grp=self.services["host_anti_affinity_1"]) + self.create_aff_grp(aff_grp=self.services["host_anti_affinity"]) + self.create_aff_grp(aff_grp=self.services["host_anti_affinity"]) vm1, hostid1 = self.create_vm_in_aff_grps([self.aff_grp[0].name]) vm2, hostid2 = self.create_vm_in_aff_grps([self.aff_grp[0].name]) @@ -915,17 +936,18 @@ class TestUpdateVMAffinityGroups(cloudstackTestCase): vm2.delete(self.api_client) #Wait for expunge interval to cleanup VM wait_for_cleanup(self.apiclient, ["expunge.delay", "expunge.interval"]) - for i in aff_grps_names: - AffinityGroup.delete(self.api_client, i) + for aff_grp in self.aff_grp: + aff_grp.delete(self.api_client) + @attr(tags=["simulator", "basic", "advanced", "multihost"]) def test_03_update_aff_grp_for_vm_with_no_aff_grp(self): """ Update the list of affinityGroups for vm which is not associated with any affinity groups. """ - self.create_aff_grp(aff_grp=self.services["host_anti_affinity_0"]) - self.create_aff_grp(aff_grp=self.services["host_anti_affinity_1"]) + self.create_aff_grp(aff_grp=self.services["host_anti_affinity"]) + self.create_aff_grp(aff_grp=self.services["host_anti_affinity"]) vm1, hostid1 = self.create_vm_in_aff_grps([]) vm2, hostid2 = self.create_vm_in_aff_grps([self.aff_grp[0].name]) @@ -945,24 +967,25 @@ class TestUpdateVMAffinityGroups(cloudstackTestCase): vm2.delete(self.api_client) #Wait for expunge interval to cleanup VM wait_for_cleanup(self.apiclient, ["expunge.delay", "expunge.interval"]) - aff_grps_names = [self.aff_grp[0].name, self.aff_grp[1].name] - for i in aff_grps_names: - AffinityGroup.delete(self.api_client, i) + aff_grps = [self.aff_grp[0], self.aff_grp[1]] + for aff_grp in aff_grps: + aff_grp.delete(self.api_client) + @unittest.skip("Skip - Failing - work in progress") + @attr(tags=["simulator", "basic", "advanced", "multihost"]) def test_04_update_aff_grp_remove_all(self): """ Update the list of Affinity Groups to empty list """ - self.create_aff_grp(aff_grp=self.services["host_anti_affinity_0"]) - self.create_aff_grp(aff_grp=self.services["host_anti_affinity_1"]) + self.create_aff_grp(aff_grp=self.services["host_anti_affinity"]) + self.create_aff_grp(aff_grp=self.services["host_anti_affinity"]) vm1, hostid1 = self.create_vm_in_aff_grps([self.aff_grp[0].name]) - aff_grps_names = [self.aff_grp[0].name, self.aff_grp[1].name] + aff_grps = [self.aff_grp[0], self.aff_grp[1]] vm1.stop(self.api_client) - vm1.update_affinity_group(self.api_client, - affinitygroupnames=[]) + vm1.update_affinity_group(self.api_client, affinitygroupids = []) vm1.start(self.api_client) list_aff_grps = AffinityGroup.list(self.api_client, @@ -972,27 +995,28 @@ class TestUpdateVMAffinityGroups(cloudstackTestCase): vm1.delete(self.api_client) #Wait for expunge interval to cleanup VM wait_for_cleanup(self.apiclient, ["expunge.delay", "expunge.interval"]) - for i in aff_grps_names: - AffinityGroup.delete(self.api_client, i) + for aff_grp in aff_grps: + aff_grp.delete(self.api_client) + @attr(tags=["simulator", "basic", "advanced", "multihost"]) def test_05_update_aff_grp_on_running_vm(self): """ Update the list of Affinity Groups on running vm """ - self.create_aff_grp(aff_grp=self.services["host_anti_affinity_0"]) - self.create_aff_grp(aff_grp=self.services["host_anti_affinity_1"]) + self.create_aff_grp(aff_grp=self.services["host_anti_affinity"]) + self.create_aff_grp(aff_grp=self.services["host_anti_affinity"]) vm1, hostid1 = self.create_vm_in_aff_grps([self.aff_grp[0].name]) - aff_grps_names = [self.aff_grp[0].name, self.aff_grp[1].name] + aff_grps = [self.aff_grp[0], self.aff_grp[1]] with self.assertRaises(Exception): vm1.update_affinity_group(self.api_client, affinitygroupnames=[]) vm1.delete(self.api_client) #Wait for expunge interval to cleanup VM wait_for_cleanup(self.apiclient, ["expunge.delay", "expunge.interval"]) - for i in aff_grps_names: - AffinityGroup.delete(self.api_client, i) + for aff_grp in aff_grps: + aff_grp.delete(self.api_client) class TestDeployVMAffinityGroups(cloudstackTestCase): @@ -1065,7 +1089,9 @@ class TestDeployVMAffinityGroups(cloudstackTestCase): if api_client == None: api_client = self.api_client if aff_grp == None: - self.services["host_anti_affinity_0"] + aff_grp = self.services["host_anti_affinity"] + + aff_grp["name"] = "aff_grp_" + random_gen(size=6) try: self.aff_grp.append(AffinityGroup.create(api_client, @@ -1118,20 +1144,19 @@ class TestDeployVMAffinityGroups(cloudstackTestCase): """ Deploy VM by aff grp name """ - - self.create_aff_grp(aff_grp=self.services["host_anti_affinity_0"]) + self.create_aff_grp(aff_grp=self.services["host_anti_affinity"]) vm1, hostid1 = self.create_vm_in_aff_grps(ag_list=[self.aff_grp[0].name]) vm1.delete(self.api_client) wait_for_cleanup(self.apiclient, ["expunge.delay", "expunge.interval"]) - AffinityGroup.delete(self.api_client, self.aff_grp[0].name) + self.aff_grp[0].delete(self.api_client) @attr(tags=["simulator", "basic", "advanced", "multihost"]) def test_03_deploy_vm_by_aff_grp_id(self): """ Deploy VM by aff grp id """ - self.create_aff_grp(aff_grp=self.services["host_anti_affinity_0"]) + self.create_aff_grp(aff_grp=self.services["host_anti_affinity"]) list_aff_grps = AffinityGroup.list(self.api_client, name=self.aff_grp[0].name) @@ -1140,7 +1165,7 @@ class TestDeployVMAffinityGroups(cloudstackTestCase): vm1.delete(self.api_client) wait_for_cleanup(self.apiclient, ["expunge.delay", "expunge.interval"]) - AffinityGroup.delete(self.api_client, self.aff_grp[0].name) + self.aff_grp[0].delete(self.api_client) @attr(tags=["simulator", "basic", "advanced", "multihost"]) def test_04_deploy_vm_anti_affinity_group(self): @@ -1150,7 +1175,7 @@ class TestDeployVMAffinityGroups(cloudstackTestCase): deploy VM1 and VM2 in the same host-anti-affinity groups Verify that the vms are deployed on separate hosts """ - self.create_aff_grp(aff_grp=self.services["host_anti_affinity_0"]) + self.create_aff_grp(aff_grp=self.services["host_anti_affinity"]) vm1, hostid1 = self.create_vm_in_aff_grps(ag_list=[self.aff_grp[0].name]) vm2, hostid2 = self.create_vm_in_aff_grps(ag_list=[self.aff_grp[0].name]) @@ -1161,14 +1186,14 @@ class TestDeployVMAffinityGroups(cloudstackTestCase): vm1.delete(self.api_client) vm2.delete(self.api_client) wait_for_cleanup(self.apiclient, ["expunge.delay", "expunge.interval"]) - AffinityGroup.delete(self.api_client, self.aff_grp[0].name) + self.aff_grp[0].delete(self.api_client) @attr(tags=["simulator", "basic", "advanced", "multihost"]) def test_05_deploy_vm_by_id(self): """ Deploy vms by affinity group id """ - self.create_aff_grp(aff_grp=self.services["host_anti_affinity_0"]) + self.create_aff_grp(aff_grp=self.services["host_anti_affinity"]) list_aff_grps = AffinityGroup.list(self.api_client, name=self.aff_grp[0].name) @@ -1183,7 +1208,7 @@ class TestDeployVMAffinityGroups(cloudstackTestCase): vm1.delete(self.api_client) vm2.delete(self.api_client) wait_for_cleanup(self.apiclient, ["expunge.delay", "expunge.interval"]) - AffinityGroup.delete(self.api_client, self.aff_grp[0].name) + self.aff_grp[0].delete(self.api_client) @attr(tags=["simulator", "basic", "advanced", "multihost"]) def test_06_deploy_vm_aff_grp_of_other_user_by_name(self): @@ -1195,7 +1220,7 @@ class TestDeployVMAffinityGroups(cloudstackTestCase): self.services["new_account"]) self.cleanup.append(self.user1) - self.create_aff_grp(aff_grp=self.services["host_anti_affinity_0"], + self.create_aff_grp(aff_grp=self.services["host_anti_affinity"], acc=self.user1.name, domainid=self.domain.id) @@ -1208,15 +1233,15 @@ class TestDeployVMAffinityGroups(cloudstackTestCase): acctType=0) self.create_aff_grp(api_client=userapiclient, - aff_grp=self.services["host_anti_affinity_1"]) + aff_grp=self.services["host_anti_affinity"]) with self.assertRaises(Exception): vm1, hostid1 = self.create_vm_in_aff_grps(api_client=userapiclient, ag_list=[self.aff_grp[0].name]) - AffinityGroup.delete(self.api_client, self.aff_grp[0].name) - AffinityGroup.delete(userapiclient, self.aff_grp[1].name) + self.aff_grp[0].delete(self.api_client) + self.aff_grp[1].delete(userapiclient) @attr(tags=["simulator", "basic", "advanced", "multihost"]) def test_07_deploy_vm_aff_grp_of_other_user_by_id(self): @@ -1228,7 +1253,7 @@ class TestDeployVMAffinityGroups(cloudstackTestCase): self.services["new_account"]) self.cleanup.append(self.user1) - self.create_aff_grp(aff_grp=self.services["host_anti_affinity_0"], + self.create_aff_grp(aff_grp=self.services["host_anti_affinity"], acc=self.user1.name, domainid=self.domain.id) @@ -1241,7 +1266,7 @@ class TestDeployVMAffinityGroups(cloudstackTestCase): acctType=0) self.create_aff_grp(api_client=userapiclient, - aff_grp=self.services["host_anti_affinity_1"]) + aff_grp=self.services["host_anti_affinity"]) list_aff_grps = AffinityGroup.list(self.api_client, name=self.aff_grp[0].name) @@ -1251,8 +1276,8 @@ class TestDeployVMAffinityGroups(cloudstackTestCase): vm1, hostid1 = self.create_vm_in_aff_grps(api_client=userapiclient, ag_ids=[list_aff_grps[0].id]) - AffinityGroup.delete(self.api_client, self.aff_grp[0].name) - AffinityGroup.delete(userapiclient, self.aff_grp[1].name) + self.aff_grp[0].delete(self.api_client) + self.aff_grp[1].delete(userapiclient) @attr(tags=["simulator", "basic", "advanced", "multihost"]) def test_08_deploy_vm_multiple_aff_grps(self): @@ -1260,8 +1285,9 @@ class TestDeployVMAffinityGroups(cloudstackTestCase): Deploy vm in multiple affinity groups """ - self.create_aff_grp(aff_grp=self.services["host_anti_affinity_0"]) - self.create_aff_grp(aff_grp=self.services["host_anti_affinity_1"]) + self.create_aff_grp(aff_grp=self.services["host_anti_affinity"]) + self.create_aff_grp(aff_grp=self.services["host_anti_affinity"]) + vm1, hostid1 = self.create_vm_in_aff_grps(ag_list=[self.aff_grp[0].name, self.aff_grp[1].name]) @@ -1279,8 +1305,8 @@ class TestDeployVMAffinityGroups(cloudstackTestCase): vm1.delete(self.api_client) wait_for_cleanup(self.apiclient, ["expunge.delay", "expunge.interval"]) - AffinityGroup.delete(self.api_client, self.aff_grp[0].name) - AffinityGroup.delete(self.api_client, self.aff_grp[1].name) + self.aff_grp[0].delete(self.api_client) + self.aff_grp[1].delete(self.api_client) @attr(tags=["simulator", "basic", "advanced", "multihost"]) def test_09_deploy_vm_multiple_aff_grps(self): @@ -1288,8 +1314,9 @@ class TestDeployVMAffinityGroups(cloudstackTestCase): Deploy multiple vms in multiple affinity groups """ - self.create_aff_grp(aff_grp=self.services["host_anti_affinity_0"]) - self.create_aff_grp(aff_grp=self.services["host_anti_affinity_1"]) + self.create_aff_grp(aff_grp=self.services["host_anti_affinity"]) + self.create_aff_grp(aff_grp=self.services["host_anti_affinity"]) + vm1, hostid1 = self.create_vm_in_aff_grps(ag_list=[self.aff_grp[0].name, self.aff_grp[1].name]) vm2, hostid2 = self.create_vm_in_aff_grps(ag_list=[self.aff_grp[0].name, @@ -1313,8 +1340,8 @@ class TestDeployVMAffinityGroups(cloudstackTestCase): vm2.delete(self.api_client) wait_for_cleanup(self.apiclient, ["expunge.delay", "expunge.interval"]) - AffinityGroup.delete(self.api_client, self.aff_grp[0].name) - AffinityGroup.delete(self.api_client, self.aff_grp[1].name) + self.aff_grp[0].delete(self.api_client) + self.aff_grp[1].delete(self.api_client) @attr(tags=["simulator", "basic", "advanced", "multihost"]) def test_10_deploy_vm_by_aff_grp_name_and_id(self): @@ -1322,7 +1349,8 @@ class TestDeployVMAffinityGroups(cloudstackTestCase): Deploy VM by aff grp name and id """ - self.create_aff_grp(aff_grp=self.services["host_anti_affinity_0"]) + self.create_aff_grp(aff_grp=self.services["host_anti_affinity"]) + list_aff_grps = AffinityGroup.list(self.api_client, name=self.aff_grp[0].name) @@ -1330,7 +1358,7 @@ class TestDeployVMAffinityGroups(cloudstackTestCase): vm1, hostid1 = self.create_vm_in_aff_grps(ag_list=[self.aff_grp[0].name], ag_ids=[list_aff_grps[0].id]) - AffinityGroup.delete(self.api_client, self.aff_grp[0].name) + self.aff_grp[0].delete(self.api_client) class TestAffinityGroupsAdminUser(cloudstackTestCase): @@ -1403,7 +1431,9 @@ class TestAffinityGroupsAdminUser(cloudstackTestCase): if api_client == None: api_client = self.api_client if aff_grp == None: - self.services["host_anti_affinity_0"] + aff_grp = self.services["host_anti_affinity"] + + aff_grp["name"] = "aff_grp_" + random_gen(size=6) try: return AffinityGroup.create(api_client, aff_grp, acc, domainid) @@ -1454,7 +1484,7 @@ class TestAffinityGroupsAdminUser(cloudstackTestCase): acctType=0) aff_grp = self.create_aff_grp(api_client=userapiclient, - aff_grp=self.services["host_anti_affinity_0"]) + aff_grp=self.services["host_anti_affinity"]) with self.assertRaises(Exception): self.create_vm_in_aff_grps(api_client=self.apiclient, ag_list=[self.aff_grp[0].name]) @@ -1471,7 +1501,7 @@ class TestAffinityGroupsAdminUser(cloudstackTestCase): domainid=self.domain.id) self.cleanup.append(self.user) - aff_grp = self.create_aff_grp(aff_grp=self.services["host_anti_affinity_0"], + aff_grp = self.create_aff_grp(aff_grp=self.services["host_anti_affinity"], acc=self.user.name, domainid=self.domain.id) aff_grp.delete(self.apiclient) @@ -1492,7 +1522,7 @@ class TestAffinityGroupsAdminUser(cloudstackTestCase): acctType=0) aff_grp = self.create_aff_grp(api_client=userapiclient, - aff_grp=self.services["host_anti_affinity_0"]) + aff_grp=self.services["host_anti_affinity"]) list_aff_grps = AffinityGroup.list(self.api_client) self.assertNotEqual(list_aff_grps, [], "Admin not able to list Affinity " diff --git a/tools/marvin/marvin/integration/lib/base.py b/tools/marvin/marvin/integration/lib/base.py index 2a2c65b4127..e4cae7fec56 100755 --- a/tools/marvin/marvin/integration/lib/base.py +++ b/tools/marvin/marvin/integration/lib/base.py @@ -571,6 +571,20 @@ class VirtualMachine: except Exception as e: raise Exception("assignVirtualMachine failed - %s" %e) + def update_affinity_group(self, apiclient, affinitygroupids=None, + affinitygroupnames=None): + """Update affinity group of a VM""" + cmd = updateVMAffinityGroup.updateVMAffinityGroupCmd() + cmd.id = self.id + + if affinitygroupids: + cmd.affinitygroupids = affinitygroupids + + if affinitygroupnames: + cmd.affinitygroupnames = affinitygroupnames + + return apiclient.updateVMAffinityGroup(cmd) + class Volume: """Manage Volume Life cycle From 6e83a54ffbf69e46a79fbaf0be63e63092385b65 Mon Sep 17 00:00:00 2001 From: Bharat Kumar Date: Fri, 2 Aug 2013 16:19:28 +0530 Subject: [PATCH 083/763] CLOUDSTACK-3753 fixed the test test_non_contigiousvlan.py Signed-off-by: Prasanna Santhanam --- test/integration/smoke/test_non_contigiousvlan.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/integration/smoke/test_non_contigiousvlan.py b/test/integration/smoke/test_non_contigiousvlan.py index 5fdc034dfb8..f1736ae4476 100644 --- a/test/integration/smoke/test_non_contigiousvlan.py +++ b/test/integration/smoke/test_non_contigiousvlan.py @@ -52,13 +52,14 @@ class TestUpdatePhysicalNetwork(cloudstackTestCase): self.network = phy_networks[0] self.networkid = phy_networks[0].id - vlan1 = self.vlan["part"][0] + self.existing_vlan = phy_networks[0].vlan + vlan1 = self.existing_vlan+","+self.vlan["part"][0] updatePhysicalNetworkResponse = self.network.update(self.apiClient, id = self.networkid, vlan = vlan1) self.assert_(updatePhysicalNetworkResponse is not None, msg="couldn't extend the physical network with vlan %s"%vlan1) self.assert_(isinstance(self.network, PhysicalNetwork)) - vlan2 = self.vlan["part"][1] + vlan2 = vlan1+","+self.vlan["part"][1] updatePhysicalNetworkResponse2 = self.network.update(self.apiClient, id = self.networkid, vlan = vlan2) self.assert_(updatePhysicalNetworkResponse2 is not None, msg="couldn't extend the physical network with vlan %s"%vlan2) @@ -80,7 +81,7 @@ class TestUpdatePhysicalNetwork(cloudstackTestCase): msg="There are no physical networks in the zone") self.network = phy_networks[0] self.networkid = phy_networks[0].id - updateResponse = self.network.update(self.apiClient, id = self.networkid, removevlan = self.vlan["full"]) + updateResponse = self.network.update(self.apiClient, id = self.networkid, vlan=self.existing_vlan) self.assert_(updateResponse.vlan.find(self.vlan["full"]) < 0, "VLAN was not removed successfully") From cb1a5a49f4efd2f4904f040bf923a6f9e2c9152e Mon Sep 17 00:00:00 2001 From: Gaurav Aradhye Date: Thu, 1 Aug 2013 22:31:00 -0400 Subject: [PATCH 084/763] CLOUDSTACK:3798: Removed wrong assert. Login was getting successful, assert was failing. Anyway it is being checked in the assert below for sesionKey Signed-off-by: Prasanna Santhanam --- test/integration/component/test_accounts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/component/test_accounts.py b/test/integration/component/test_accounts.py index 9c3d231a0e0..cf18bef84da 100644 --- a/test/integration/component/test_accounts.py +++ b/test/integration/component/test_accounts.py @@ -1229,7 +1229,7 @@ class TestUserLogin(cloudstackTestCase): username=self.account.name, password=self.services["account"]["password"] ) - self.assertEqual(respose, None, "Login response should not be none") + self.debug("Login API response: %s" % respose) self.assertNotEqual( From 59b23fe1f416bfa22d27a277c88edb3e7eca672e Mon Sep 17 00:00:00 2001 From: Wido den Hollander Date: Thu, 1 Aug 2013 11:08:56 +0200 Subject: [PATCH 085/763] CLOUDSTACK-4004: Fix attaching a newly allocated RBD image to an Instance Signed-off-by: Wido den Hollander --- .../hypervisor/kvm/resource/LibvirtVMDef.java | 2 +- .../kvm/storage/KVMStorageProcessor.java | 16 ++++++++++++---- .../kvm/storage/LibvirtStorageAdaptor.java | 4 ++-- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java index 9a3bef9f381..6aaabc5be13 100644 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java @@ -384,7 +384,7 @@ public class LibvirtVMDef { } } - enum diskProtocol { + public enum diskProtocol { RBD("rbd"), SHEEPDOG("sheepdog"); String _diskProtocol; diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java index 35cdaf34186..783c65e078d 100644 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java @@ -67,6 +67,7 @@ import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; import com.cloud.hypervisor.kvm.resource.LibvirtConnection; import com.cloud.hypervisor.kvm.resource.LibvirtDomainXMLParser; import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.DiskDef; +import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.DiskDef.diskProtocol; import com.cloud.storage.JavaStorageLayer; import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.Storage.StoragePoolType; @@ -626,6 +627,7 @@ public class KVMStorageProcessor implements StorageProcessor { List disks = null; Domain dm = null; DiskDef diskdef = null; + KVMStoragePool attachingPool = attachingDisk.getPool(); try { if (!attach) { dm = conn.domainLookupByName(vmName); @@ -646,11 +648,17 @@ public class KVMStorageProcessor implements StorageProcessor { } } else { diskdef = new DiskDef(); - if (attachingDisk.getFormat() == PhysicalDiskFormat.QCOW2) { - diskdef.defFileBasedDisk(attachingDisk.getPath(), devId, DiskDef.diskBus.VIRTIO, - DiskDef.diskFmtType.QCOW2); + if (attachingPool.getType() == StoragePoolType.RBD) { + diskdef.defNetworkBasedDisk(attachingDisk.getPath(), + attachingPool.getSourceHost(), attachingPool.getSourcePort(), + attachingPool.getAuthUserName(), attachingPool.getUuid(), devId, + DiskDef.diskBus.VIRTIO, diskProtocol.RBD); + } else if (attachingDisk.getFormat() == PhysicalDiskFormat.QCOW2) { + diskdef.defFileBasedDisk(attachingDisk.getPath(), devId, + DiskDef.diskBus.VIRTIO, DiskDef.diskFmtType.QCOW2); } else if (attachingDisk.getFormat() == PhysicalDiskFormat.RAW) { - diskdef.defBlockBasedDisk(attachingDisk.getPath(), devId, DiskDef.diskBus.VIRTIO); + diskdef.defBlockBasedDisk(attachingDisk.getPath(), devId, + DiskDef.diskBus.VIRTIO); } } diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java index a9baa52d71c..e77916255e9 100644 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java @@ -634,7 +634,7 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { format = PhysicalDiskFormat.RAW; try { - s_logger.info("Creating RBD image " + pool.getSourcePort() + "/" + name + " with size " + size); + s_logger.info("Creating RBD image " + pool.getSourceDir() + "/" + name + " with size " + size); Rados r = new Rados(pool.getAuthUserName()); r.confSet("mon_host", pool.getSourceHost() + ":" + pool.getSourcePort()); @@ -653,7 +653,7 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { throw new CloudRuntimeException(e.toString()); } - volPath = name; + volPath = pool.getSourceDir() + "/" + name; volName = name; volCapacity = size; volAllocation = size; From 1e458b4756611cd15c53278ddf344ba05fe4f623 Mon Sep 17 00:00:00 2001 From: radhikap Date: Fri, 2 Aug 2013 17:01:42 +0530 Subject: [PATCH 086/763] https://issues.apache.org/jira/browse/CLOUDSTACK-2685 --- docs/en-US/creating-network-offerings.xml | 15 +- docs/en-US/egress-firewall-rule.xml | 168 +++++++++++++++------- docs/en-US/ip-forwarding-firewalling.xml | 9 +- docs/en-US/vnmc-cisco.xml | 12 +- 4 files changed, 138 insertions(+), 66 deletions(-) diff --git a/docs/en-US/creating-network-offerings.xml b/docs/en-US/creating-network-offerings.xml index 6e25b27e9ad..4f75781c3cb 100644 --- a/docs/en-US/creating-network-offerings.xml +++ b/docs/en-US/creating-network-offerings.xml @@ -241,7 +241,7 @@ For information on Elastic IP, see . - Redundant router capability. Available only when + Redundant router capability: Available only when Virtual Router is selected as the Source NAT provider. Select this option if you want to use two virtual routers in the network for uninterrupted connection: one operating as the master virtual router and the other as the backup. The master virtual router @@ -251,7 +251,7 @@ reliability if one host is down. - Conserve mode. Indicate whether to use conserve + Conserve mode: Indicate whether to use conserve mode. In this mode, network resources are allocated only when the first virtual machine starts in the network. When conservative mode is off, the public IP can only be used for a single service. For example, a public IP used for a port forwarding rule cannot be @@ -264,9 +264,18 @@ - Tags. Network tag to specify which physical network + Tags: Network tag to specify which physical network to use. + + Default egress policy: Configure the default policy + for firewall egress rules. Options are Allow and Deny. Default is Allow if no egress + policy is specified, which indicates that all the egress traffic is accepted when a + guest network is created from this offering. + To block the egress traffic for a guest network, select Deny. In this case, when you + configure an egress rules for an isolated guest network, rules are added to allow the + specified traffic. + diff --git a/docs/en-US/egress-firewall-rule.xml b/docs/en-US/egress-firewall-rule.xml index 9b45e2e02a2..148b6d6c18a 100644 --- a/docs/en-US/egress-firewall-rule.xml +++ b/docs/en-US/egress-firewall-rule.xml @@ -19,7 +19,7 @@ under the License. -->
- Creating Egress Firewall Rules in an Advanced Zone + Egress Firewall Rules in an Advanced Zone The egress traffic originates from a private network to a public network, such as the Internet. By default, the egress traffic is blocked, so no outgoing traffic is allowed from a guest network to the Internet. However, you can control the egress traffic in an Advanced zone @@ -27,7 +27,7 @@ to the rule is allowed and the remaining traffic is blocked. When all the firewall rules are removed the default policy, Block, is applied. Egress firewall rules are supported on Juniper SRX and virtual router. - + The egress firewall rules are not supported on shared networks. Consider the following scenarios to apply egress firewall rules: @@ -44,55 +44,117 @@ specified for TCP, UDP or for ICMP type and code. - To configure an egress firewall rule: - - - Log in to the &PRODUCT; UI as an administrator or end user. - - - In the left navigation, choose Network. - - - In Select view, choose Guest networks, then click the Guest network you want. - - - To add an egress rule, click the Egress rules tab and fill out the following fields to - specify what type of traffic is allowed to be sent out of VM instances in this guest - network: - - - - - - egress-firewall-rule.png: adding an egress firewall rule - - - - - CIDR: (Add by CIDR only) To send traffic only to - the IP addresses within a particular address block, enter a CIDR or a comma-separated - list of CIDRs. The CIDR is the base IP address of the destination. For example, - 192.168.0.0/22. To allow all CIDRs, set to 0.0.0.0/0. - - - Protocol: The networking protocol that VMs uses to - send outgoing traffic. The TCP and UDP protocols are typically used for data exchange - and end-user communications. The ICMP protocol is typically used to send error messages - or network monitoring data. - - - Start Port, End Port: (TCP, UDP only) A range of - listening ports that are the destination for the outgoing traffic. If you are opening a - single port, use the same number in both fields. - - - ICMP Type, ICMP Code: (ICMP only) The type of - message and error code that are sent. - - - - - Click Add. - - +
+ Configuring an Egress Firewall Rule + + + Log in to the &PRODUCT; UI as an administrator or end user. + + + In the left navigation, choose Network. + + + In Select view, choose Guest networks, then click the Guest network you want. + + + To add an egress rule, click the Egress rules tab and fill out the following fields to + specify what type of traffic is allowed to be sent out of VM instances in this guest + network: + + + + + + egress-firewall-rule.png: adding an egress firewall rule + + + + + CIDR: (Add by CIDR only) To send traffic only to + the IP addresses within a particular address block, enter a CIDR or a comma-separated + list of CIDRs. The CIDR is the base IP address of the destination. For example, + 192.168.0.0/22. To allow all CIDRs, set to 0.0.0.0/0. + + + Protocol: The networking protocol that VMs uses + to send outgoing traffic. The TCP and UDP protocols are typically used for data + exchange and end-user communications. The ICMP protocol is typically used to send + error messages or network monitoring data. + + + Start Port, End Port: (TCP, UDP only) A range of + listening ports that are the destination for the outgoing traffic. If you are opening + a single port, use the same number in both fields. + + + ICMP Type, ICMP Code: (ICMP only) The type of + message and error code that are sent. + + + + + Click Add. + + +
+
+ Changing the Default Egress Policy + You can configure the default policy of egress firewall rules in Isolated Advanced + networks. Use the create network offering option to determine whether the default policy + should be block or allow all the traffic to the public network from a guest network. If no + policy is specified, by default all the traffic is allowed from the guest network that you + create by using this network offering. + You have two options: Allow and Deny. + + Allow + If you select Allow for a network offering, by default egress traffic is allowed. + However, when an egress rule is configured for a guest network, rules are applied to block + the specified traffic and rest are allowed. If no egress rules are configured for the + network, egress traffic is accepted. + + + Deny + If you select Deny for a network offering, by default egress traffic for the guest + network is blocked. However, when an egress rules is configured for a guest network, rules + are applied to allow the specified traffic. While implementing a guest network, &PRODUCT; + adds the firewall egress rule specific to the default egress policy for the guest + network. + + This feature is supported only on virtual router and Juniper SRX. + + + Create a network offering with your desirable default egress policy: + + + Log in with admin privileges to the &PRODUCT; UI. + + + In the left navigation bar, click Service Offerings. + + + In Select Offering, choose Network Offering. + + + Click Add Network Offering. + + + In the dialog, make necessary choices, including firewall provider. + + + In the Default egress policy field, specify the behaviour. + + + Click OK. + + + + + Create an isolated network by using this network offering. + Based on your selection, the network will have the egress public traffic blocked or + allowed. + + + On upgrade existing network offerings with firewall service providers will have the + default egress policy DENY. +
diff --git a/docs/en-US/ip-forwarding-firewalling.xml b/docs/en-US/ip-forwarding-firewalling.xml index d7a24571429..d1beb2eb0f2 100644 --- a/docs/en-US/ip-forwarding-firewalling.xml +++ b/docs/en-US/ip-forwarding-firewalling.xml @@ -20,15 +20,16 @@ -->
IP Forwarding and Firewalling - By default, all incoming traffic to the public IP address is rejected. - All outgoing traffic from the guests is also blocked by default. - To allow outgoing traffic, follow the procedure in . + By default, all incoming traffic to the public IP address is rejected. All outgoing traffic + from the guests is also blocked by default. + To allow outgoing traffic, follow the procedure in . To allow incoming traffic, users may set up firewall rules and/or port forwarding rules. For example, you can use a firewall rule to open a range of ports on the public IP address, such as 33 through 44. Then use port forwarding rules to direct traffic from individual ports within that range to specific ports on user VMs. For example, one port forwarding rule could route incoming traffic on the public IP's port 33 to port 100 on one user VM's private IP. - +
diff --git a/docs/en-US/vnmc-cisco.xml b/docs/en-US/vnmc-cisco.xml index 809c1517b8e..3d201606625 100644 --- a/docs/en-US/vnmc-cisco.xml +++ b/docs/en-US/vnmc-cisco.xml @@ -106,12 +106,12 @@
Guidelines When a guest network is created with Cisco VNMC firewall provider, an additional public - IP is by default acquired along with the Source NAT IP. The Source NAT IP is used for the - ASA outside interface, whereas the addition IP is used to workaround an ASA limitation. - Ensure that this additional public IP is not released. You can identify this IP as soon as - the network is in implemented state and before acquiring any further public IPs. The - additional IP is the one that is not marked as Source NAT. You can find the IP used for the - ASA outside interface by looking at the Cisco VNMC used in your guest network. + IP is acquired along with the Source NAT IP. The Source NAT IP is used for the ASA outside + interface, whereas the additional IP is used to workaround an ASA limitation. Ensure that + this additional public IP is not released. You can identify this IP as soon as the network + is in implemented state and before acquiring any further public IPs. The additional IP is + the one that is not marked as Source NAT. You can find the IP used for the ASA outside + interface by looking at the Cisco VNMC used in your guest network.
Using Cisco ASA 1000v Services From aa302387d100074cc5294998565b9382ee43aec1 Mon Sep 17 00:00:00 2001 From: radhikap Date: Fri, 2 Aug 2013 17:11:14 +0530 Subject: [PATCH 087/763] minor edit --- docs/en-US/egress-firewall-rule.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en-US/egress-firewall-rule.xml b/docs/en-US/egress-firewall-rule.xml index 148b6d6c18a..68d989800da 100644 --- a/docs/en-US/egress-firewall-rule.xml +++ b/docs/en-US/egress-firewall-rule.xml @@ -19,7 +19,7 @@ under the License. -->
- Egress Firewall Rules in an Advanced Zone + Egress Firewall Rules in Advanced Zone The egress traffic originates from a private network to a public network, such as the Internet. By default, the egress traffic is blocked, so no outgoing traffic is allowed from a guest network to the Internet. However, you can control the egress traffic in an Advanced zone From 60585b51819e4391373d71c63138e36f0a3c5898 Mon Sep 17 00:00:00 2001 From: Bharat Kumar Date: Fri, 2 Aug 2013 16:38:27 +0530 Subject: [PATCH 088/763] [PATCH] upgrade to ASF 4.2 cloud Database Schema Inconsistencies --- .../schema/src/com/cloud/storage/VolumeVO.java | 11 +++++++++++ server/src/com/cloud/storage/VolumeManager.java | 2 +- .../src/com/cloud/storage/VolumeManagerImpl.java | 14 ++++++++++++-- .../com/cloud/template/TemplateManagerImpl.java | 16 ++++++++++++++++ .../com/cloud/vm/VirtualMachineManagerImpl.java | 4 ++-- setup/db/db/schema-307to410.sql | 1 + setup/db/db/schema-410to420.sql | 3 ++- 7 files changed, 45 insertions(+), 6 deletions(-) diff --git a/engine/schema/src/com/cloud/storage/VolumeVO.java b/engine/schema/src/com/cloud/storage/VolumeVO.java index 1445e99d727..bbecf0346c3 100755 --- a/engine/schema/src/com/cloud/storage/VolumeVO.java +++ b/engine/schema/src/com/cloud/storage/VolumeVO.java @@ -152,6 +152,9 @@ public class VolumeVO implements Volume { @Column(name = "vm_snapshot_chain_size") private Long vmSnapshotChainSize; + + @Column(name = "iso_id") + private long isoId; @Transient // @Column(name="reservation") @@ -561,4 +564,12 @@ public class VolumeVO implements Volume { public Long getVmSnapshotChainSize(){ return this.vmSnapshotChainSize; } + + public Long getIsoId() { + return this.isoId; + } + + public void setIsoId(long isoId) { + this.isoId =isoId; + } } diff --git a/server/src/com/cloud/storage/VolumeManager.java b/server/src/com/cloud/storage/VolumeManager.java index 2e44a3c35b3..c8adaebe764 100644 --- a/server/src/com/cloud/storage/VolumeManager.java +++ b/server/src/com/cloud/storage/VolumeManager.java @@ -78,7 +78,7 @@ public interface VolumeManager extends VolumeApiService { void destroyVolume(VolumeVO volume); - DiskProfile allocateRawVolume(Type type, String name, DiskOfferingVO offering, Long size, VMInstanceVO vm, Account owner); + DiskProfile allocateRawVolume(Type type, String name, DiskOfferingVO offering, Long size, VMInstanceVO vm, VMTemplateVO template, Account owner); @Override Volume attachVolumeToVM(AttachVolumeCmd command); diff --git a/server/src/com/cloud/storage/VolumeManagerImpl.java b/server/src/com/cloud/storage/VolumeManagerImpl.java index abbc2f0f258..c4846668580 100644 --- a/server/src/com/cloud/storage/VolumeManagerImpl.java +++ b/server/src/com/cloud/storage/VolumeManagerImpl.java @@ -31,7 +31,13 @@ import java.util.concurrent.ExecutionException; import javax.inject.Inject; import javax.naming.ConfigurationException; -import com.cloud.utils.*; +import com.cloud.utils.DateUtil; +import com.cloud.utils.EnumUtils; +import com.cloud.utils.NumbersUtil; +import com.cloud.utils.Pair; +import com.cloud.utils.UriUtils; +import com.cloud.utils.db.SearchCriteria; +import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -1380,7 +1386,8 @@ public class VolumeManagerImpl extends ManagerBase implements VolumeManager { @Override public DiskProfile allocateRawVolume(Type type, - String name, DiskOfferingVO offering, Long size, VMInstanceVO vm, Account owner) { + String name, DiskOfferingVO offering, Long size, VMInstanceVO vm, VMTemplateVO template, Account owner) { + Long isoId=null; if (size == null) { size = offering.getDiskSize(); } else { @@ -1398,6 +1405,9 @@ public class VolumeManagerImpl extends ManagerBase implements VolumeManager { } else { vol.setDeviceId(1l); } + if (template.getFormat() == ImageFormat.ISO) { + vol.setIsoId(template.getId()); + } vol.setFormat(getSupportedImageFormatForCluster(vm.getHypervisorType())); vol = _volsDao.persist(vol); diff --git a/server/src/com/cloud/template/TemplateManagerImpl.java b/server/src/com/cloud/template/TemplateManagerImpl.java index 8ccc567aebf..efef80b4208 100755 --- a/server/src/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/com/cloud/template/TemplateManagerImpl.java @@ -1376,6 +1376,22 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, this._tmpltZoneDao.persist(templateZone); privateTemplate = this._tmpltDao.findById(templateId); + if (snapshotId != null) { + //getting the prent volume + long parentVolumeId=_snapshotDao.findById(snapshotId).getVolumeId(); + VolumeVO parentVolume = _volumeDao.findById(parentVolumeId); + if (parentVolume.getIsoId() != null) { + privateTemplate.setSourceTemplateId(parentVolume.getIsoId()); + _tmpltDao.update(privateTemplate.getId(), privateTemplate); + } + } + else if (volumeId != null) { + VolumeVO parentVolume = _volumeDao.findById(volumeId); + if (parentVolume.getIsoId() != null) { + privateTemplate.setSourceTemplateId(parentVolume.getIsoId()); + _tmpltDao.update(privateTemplate.getId(), privateTemplate); + } + } TemplateDataStoreVO srcTmpltStore = this._tmplStoreDao.findByStoreTemplate(store.getId(), templateId); UsageEventVO usageEvent = new UsageEventVO(EventTypes.EVENT_TEMPLATE_CREATE, privateTemplate.getAccountId(), zoneId, privateTemplate.getId(), privateTemplate.getName(), null, privateTemplate.getSourceTemplateId(), srcTmpltStore.getPhysicalSize(), privateTemplate.getSize()); diff --git a/server/src/com/cloud/vm/VirtualMachineManagerImpl.java b/server/src/com/cloud/vm/VirtualMachineManagerImpl.java index 6d355392915..cf24502c6fb 100755 --- a/server/src/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/server/src/com/cloud/vm/VirtualMachineManagerImpl.java @@ -388,7 +388,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } if (template.getFormat() == ImageFormat.ISO) { - volumeMgr.allocateRawVolume(Type.ROOT, "ROOT-" + vm.getId(), rootDiskOffering.first(), rootDiskOffering.second(), vm, owner); + volumeMgr.allocateRawVolume(Type.ROOT, "ROOT-" + vm.getId(), rootDiskOffering.first(), rootDiskOffering.second(), vm, template, owner); } else if (template.getFormat() == ImageFormat.BAREMETAL) { // Do nothing } else { @@ -396,7 +396,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } for (Pair offering : dataDiskOfferings) { - volumeMgr.allocateRawVolume(Type.DATADISK, "DATA-" + vm.getId(), offering.first(), offering.second(), vm, owner); + volumeMgr.allocateRawVolume(Type.DATADISK, "DATA-" + vm.getId(), offering.first(), offering.second(), vm, template, owner); } txn.commit(); diff --git a/setup/db/db/schema-307to410.sql b/setup/db/db/schema-307to410.sql index be6c17da148..af296abbe25 100644 --- a/setup/db/db/schema-307to410.sql +++ b/setup/db/db/schema-307to410.sql @@ -1572,3 +1572,4 @@ ALTER TABLE `cloud_usage`.`usage_storage` CHANGE COLUMN `virtual_size` `virtual_ ALTER TABLE `cloud_usage`.`cloud_usage` CHANGE COLUMN `virtual_size` `virtual_size1` bigint unsigned; ALTER TABLE `cloud`.`network_offerings` CHANGE COLUMN `concurrent_connections` `concurrent_connections1` int(10) unsigned COMMENT 'Load Balancer(haproxy) maximum number of concurrent connections(global max)'; +ALTER TABLE `cloud`.`volumes` DROP COLUMN `iso_id`; \ No newline at end of file diff --git a/setup/db/db/schema-410to420.sql b/setup/db/db/schema-410to420.sql index 6fdf3fea267..aeacff3928a 100644 --- a/setup/db/db/schema-410to420.sql +++ b/setup/db/db/schema-410to420.sql @@ -1791,7 +1791,7 @@ CREATE VIEW `cloud`.`volume_view` AS left join `cloud`.`cluster` ON storage_pool.cluster_id = cluster.id left join - `cloud`.`vm_template` ON volumes.template_id = vm_template.id + `cloud`.`vm_template` ON volumes.template_id = vm_template.id OR volumes.iso_id = vm_template.id left join `cloud`.`resource_tags` ON resource_tags.resource_id = volumes.id and resource_tags.resource_type = 'Volume' @@ -2209,6 +2209,7 @@ CREATE TABLE `cloud_usage`.`usage_vmsnapshot` ( ) ENGINE=InnoDB CHARSET=utf8; ALTER TABLE volumes ADD COLUMN vm_snapshot_chain_size bigint(20) unsigned; +ALTER TABLE volumes ADD COLUMN iso_id bigint(20) unsigned; INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Advanced', 'DEFAULT', 'management-server', 'kvm.ssh.to.agent', 'true', 'Specify whether or not the management server is allowed to SSH into KVM Agents'); From aead26f7699b0ee475db23a8a8b1bc163c8eefd8 Mon Sep 17 00:00:00 2001 From: Kishan Kavala Date: Fri, 2 Aug 2013 18:03:54 +0530 Subject: [PATCH 089/763] CLOUDSTACK-4005: Do not allow snapshot on zone wide primary storage when KVM.snapshot.enabled is set to false --- .../storage/snapshot/SnapshotManagerImpl.java | 37 +++++++++++++------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/server/src/com/cloud/storage/snapshot/SnapshotManagerImpl.java b/server/src/com/cloud/storage/snapshot/SnapshotManagerImpl.java index 2b291d3538a..b639aa8cdef 100755 --- a/server/src/com/cloud/storage/snapshot/SnapshotManagerImpl.java +++ b/server/src/com/cloud/storage/snapshot/SnapshotManagerImpl.java @@ -929,21 +929,34 @@ public class SnapshotManagerImpl extends ManagerBase implements SnapshotManager, } private boolean supportedByHypervisor(VolumeInfo volume) { - StoragePool storagePool = (StoragePool)volume.getDataStore(); - ClusterVO cluster = _clusterDao.findById(storagePool.getClusterId()); - if (cluster != null && cluster.getHypervisorType() == HypervisorType.Ovm) { + HypervisorType hypervisorType; + StoragePoolVO storagePool = _storagePoolDao.findById(volume.getDataStore().getId()); + ScopeType scope = storagePool.getScope(); + if (scope.equals(ScopeType.ZONE)) { + hypervisorType = storagePool.getHypervisor(); + } else { + hypervisorType = volume.getHypervisorType(); + } + + if (hypervisorType.equals(HypervisorType.Ovm)) { throw new InvalidParameterValueException("Ovm won't support taking snapshot"); } - if (volume.getHypervisorType().equals(HypervisorType.KVM)) { - List hosts = _resourceMgr.listAllHostsInCluster(cluster.getId()); - if (hosts != null && !hosts.isEmpty()) { - HostVO host = hosts.get(0); - if (!hostSupportSnapsthot(host)) { - throw new CloudRuntimeException("KVM Snapshot is not supported on cluster: " + host.getId()); - } - } - } + if (hypervisorType.equals(HypervisorType.KVM)) { + List hosts = null; + if(scope.equals(ScopeType.CLUSTER)){ + ClusterVO cluster = _clusterDao.findById(storagePool.getClusterId()); + hosts = _resourceMgr.listAllHostsInCluster(cluster.getId()); + } else if (scope.equals(ScopeType.ZONE)){ + hosts = _resourceMgr.listAllUpAndEnabledHostsInOneZoneByHypervisor(hypervisorType, volume.getDataCenterId()); + } + if (hosts != null && !hosts.isEmpty()) { + HostVO host = hosts.get(0); + if (!hostSupportSnapsthot(host)) { + throw new CloudRuntimeException("KVM Snapshot is not supported: " + host.getId()); + } + } + } // if volume is attached to a vm in destroyed or expunging state; disallow if (volume.getInstanceId() != null) { From b4033d5ada754c2e031f71c472c3732861c4fbf0 Mon Sep 17 00:00:00 2001 From: Prasanna Santhanam Date: Fri, 2 Aug 2013 18:15:52 +0530 Subject: [PATCH 090/763] CLOUDSTACK-3926: Add isportable=True explicitly Signed-off-by: Prasanna Santhanam (cherry picked from commit 025ab82b043c9bb9ce59eba884d81c89c81bcb8e) --- test/integration/smoke/test_portable_publicip.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/integration/smoke/test_portable_publicip.py b/test/integration/smoke/test_portable_publicip.py index 73f745cfa4a..0faed7163cb 100644 --- a/test/integration/smoke/test_portable_publicip.py +++ b/test/integration/smoke/test_portable_publicip.py @@ -69,10 +69,10 @@ class Services: "displaytext": "Test Network", }, "ostype": 'CentOS 5.3 (64-bit)', - "gateway" : "10.1.1.1", + "gateway" : "172.1.1.1", "netmask" : "255.255.255.0", - "startip" : "10.1.1.10", - "endip" : "10.1.1.20", + "startip" : "172.1.1.10", + "endip" : "172.1.1.20", "regionid" : "1", "vlan" :"10", "isportable" : "true", @@ -230,7 +230,7 @@ class TestPortablePublicIPAcquire(cloudstackTestCase): ) ip_address = PublicIPAddress.create(self.api_client, self.account.name, - self.zone.id, self.account.domainid, True) + self.zone.id, self.account.domainid, isportable=True) ip_address.delete(self.api_client) self.portable_ip_range.delete(self.apiclient) From 0951739e32fa6e2c9cee1bd1bca2817c2536ef50 Mon Sep 17 00:00:00 2001 From: Bharat Kumar Date: Fri, 2 Aug 2013 19:03:09 +0530 Subject: [PATCH 091/763] fix to add the the overcommit details of the vms in uservm details table when decreasing the overcommit to one. Signed-off-by: Abhinandan Prateek --- server/src/com/cloud/vm/VirtualMachineManagerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/com/cloud/vm/VirtualMachineManagerImpl.java b/server/src/com/cloud/vm/VirtualMachineManagerImpl.java index cf24502c6fb..b33ee49cb68 100755 --- a/server/src/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/server/src/com/cloud/vm/VirtualMachineManagerImpl.java @@ -906,7 +906,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac _uservmDetailsDao.persist(vmDetail_cpu); _uservmDetailsDao.persist(vmDetail_ram); } - else if (((Float.parseFloat(cluster_detail_cpu.getValue()) > 1f || Float.parseFloat(cluster_detail_ram.getValue()) > 1f))) { + else if (_uservmDetailsDao.findDetail(vm.getId(),"cpuOvercommitRatio") != null) { UserVmDetailVO vmDetail_cpu = _uservmDetailsDao.findDetail(vm.getId(), "cpuOvercommitRatio"); vmDetail_cpu.setValue(cluster_detail_cpu.getValue()); UserVmDetailVO vmDetail_ram = _uservmDetailsDao.findDetail(vm.getId(), "memoryOvercommitRatio"); From ed08f0f55174d449238c0e3cac93e022097045ae Mon Sep 17 00:00:00 2001 From: Alena Prokharchyk Date: Fri, 2 Aug 2013 09:01:27 -0700 Subject: [PATCH 092/763] CLOUDSTACK-4022: listProjects - do searchIncludingRemoved when getting info of corresponding projectAccount because when the project is being removed, its account is marked as Removed first, but we might still want to retrieve its info --- server/src/com/cloud/api/query/dao/ProjectJoinDaoImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/com/cloud/api/query/dao/ProjectJoinDaoImpl.java b/server/src/com/cloud/api/query/dao/ProjectJoinDaoImpl.java index 6c98f8c95d1..616f256db3a 100644 --- a/server/src/com/cloud/api/query/dao/ProjectJoinDaoImpl.java +++ b/server/src/com/cloud/api/query/dao/ProjectJoinDaoImpl.java @@ -91,7 +91,7 @@ public class ProjectJoinDaoImpl extends GenericDaoBase impl } //set resource limit/count information for the project (by getting the info of the project's account) - Account account = _accountDao.findById(proj.getProjectAccountId()); + Account account = _accountDao.findByIdIncludingRemoved(proj.getProjectAccountId()); AccountJoinVO accountJn = ApiDBUtils.newAccountView(account); _accountJoinDao.setResourceLimits(accountJn, false, response); From 33f5bc4bb34463a304e29a1584241eccdf2918dd Mon Sep 17 00:00:00 2001 From: Prasanna Santhanam Date: Fri, 2 Aug 2013 22:04:51 +0530 Subject: [PATCH 093/763] make name=value when calling publicipaddress Signed-off-by: Prasanna Santhanam (cherry picked from commit 04cc1be58ad4c768709bd975845e8a6e7d1c50ee) --- test/integration/component/test_usage.py | 32 ++++++++++---------- test/integration/component/test_vpn_users.py | 8 ++--- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/test/integration/component/test_usage.py b/test/integration/component/test_usage.py index a3779e4dc2f..5a425590c41 100644 --- a/test/integration/component/test_usage.py +++ b/test/integration/component/test_usage.py @@ -336,10 +336,10 @@ class TestPublicIPUsage(cloudstackTestCase): cls.public_ip = PublicIPAddress.create( cls.api_client, - cls.virtual_machine.account, - cls.virtual_machine.zoneid, - cls.virtual_machine.domainid, - cls.services["server"] + accountid=cls.virtual_machine.account, + zoneid=cls.virtual_machine.zoneid, + domainid=cls.virtual_machine.domainid, + services=cls.services["server"] ) cls._cleanup = [ cls.service_offering, @@ -962,10 +962,10 @@ class TestLBRuleUsage(cloudstackTestCase): ) cls.public_ip_1 = PublicIPAddress.create( cls.api_client, - cls.virtual_machine.account, - cls.virtual_machine.zoneid, - cls.virtual_machine.domainid, - cls.services["server"] + accountid=cls.virtual_machine.account, + zoneid=cls.virtual_machine.zoneid, + domainid=cls.virtual_machine.domainid, + services=cls.services["server"] ) cls._cleanup = [ cls.service_offering, @@ -1291,10 +1291,10 @@ class TestNatRuleUsage(cloudstackTestCase): ) cls.public_ip_1 = PublicIPAddress.create( cls.api_client, - cls.virtual_machine.account, - cls.virtual_machine.zoneid, - cls.virtual_machine.domainid, - cls.services["server"] + accountid=cls.virtual_machine.account, + zoneid=cls.virtual_machine.zoneid, + domainid=cls.virtual_machine.domainid, + services=cls.services["server"] ) cls._cleanup = [ cls.service_offering, @@ -1454,10 +1454,10 @@ class TestVpnUsage(cloudstackTestCase): ) cls.public_ip = PublicIPAddress.create( cls.api_client, - cls.virtual_machine.account, - cls.virtual_machine.zoneid, - cls.virtual_machine.domainid, - cls.services["server"] + accountid=cls.virtual_machine.account, + zoneid=cls.virtual_machine.zoneid, + domainid=cls.virtual_machine.domainid, + services=cls.services["server"] ) cls._cleanup = [ cls.service_offering, diff --git a/test/integration/component/test_vpn_users.py b/test/integration/component/test_vpn_users.py index e327624cdad..9ee907bd94c 100644 --- a/test/integration/component/test_vpn_users.py +++ b/test/integration/component/test_vpn_users.py @@ -149,10 +149,10 @@ class TestVPNUsers(cloudstackTestCase): ) self.public_ip = PublicIPAddress.create( self.apiclient, - self.virtual_machine.account, - self.virtual_machine.zoneid, - self.virtual_machine.domainid, - self.services["virtual_machine"] + accountid=self.virtual_machine.account, + zoneid=self.virtual_machine.zoneid, + domainid=self.virtual_machine.domainid, + services=self.services["virtual_machine"] ) return except cloudstackAPIException as e: From 6f63817d07746bf4674e3686e73cafc4bf852a2c Mon Sep 17 00:00:00 2001 From: Prasanna Santhanam Date: Fri, 2 Aug 2013 22:10:35 +0530 Subject: [PATCH 094/763] vpn tags are not supported resource types Signed-off-by: Prasanna Santhanam --- test/integration/component/test_tags.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/integration/component/test_tags.py b/test/integration/component/test_tags.py index 67aa99ea06a..6776c8c11fa 100644 --- a/test/integration/component/test_tags.py +++ b/test/integration/component/test_tags.py @@ -697,6 +697,7 @@ class TestResourceTags(cloudstackTestCase): # 1. Enable the VPN # 2. create Tag on VPN rule using CreateTag API # 3. Delete the VPN rule + self.skipTest("VPN resource tags are unsupported in 4.0") self.debug("Fetching the network details for account: %s" % self.account.name) From 80ae4830ae63ab73a06ab033bc3feb3dd701cbb2 Mon Sep 17 00:00:00 2001 From: Brian Federle Date: Fri, 2 Aug 2013 09:51:55 -0700 Subject: [PATCH 095/763] CLOUDSTACK-4010: Events/alerts: Pass start date / end date Original submitter: Sanjay Tripathi Reviewed by: Brian Federle --- ui/scripts/events.js | 93 +++++++++++++++++++++++++++++++------------- 1 file changed, 67 insertions(+), 26 deletions(-) diff --git a/ui/scripts/events.js b/ui/scripts/events.js index 873998ef769..d2e9aa49b55 100644 --- a/ui/scripts/events.js +++ b/ui/scripts/events.js @@ -75,8 +75,13 @@ label: 'By event type', docID: 'helpEventsDeleteType' }, - date: { - label: 'By date (older than)', + startdate: { + label: 'By date (start date)', + docID: 'helpEventsDeleteDate', + isDatepicker: true + }, + enddate: { + label: 'By date (end date)', docID: 'helpEventsDeleteDate', isDatepicker: true } @@ -91,9 +96,14 @@ type: args.data.type }); - if (args.data.date != "") + if (args.data.startdate != "") $.extend(data, { - olderthan: args.data.date + startdate: args.data.startdate + }); + + if (args.data.enddate != "") + $.extend(data, { + enddate: args.data.enddate }); $.ajax({ @@ -132,11 +142,16 @@ label: 'By event type', docID: 'helpEventsArchiveType' }, - date: { - label: 'By date (older than)', + startdate: { + label: 'By date (start date)', docID: 'helpEventsArchiveDate', isDatepicker: true }, + enddate: { + label: 'By date (end date)', + docID: 'helpEventsArchiveDate', + isDatepicker: true + } } }, action: function(args) { @@ -147,9 +162,14 @@ type: args.data.type }); - if (args.data.date != "") + if (args.data.startdate != "") $.extend(data, { - olderthan: args.data.date + startdate: args.data.startdate + }); + + if (args.data.enddate != "") + $.extend(data, { + enddate: args.data.enddate }); $.ajax({ @@ -250,16 +270,16 @@ } } /* - , - startdate: { - label: 'Start Date', - isDatepicker: true - }, - enddate: { - label: 'End Date', - isDatepicker: true - } - */ + , + startdate: { + label: 'Start Date', + isDatepicker: true + }, + enddate: { + label: 'End Date', + isDatepicker: true + } + */ }, dataProvider: function(args) { @@ -424,8 +444,13 @@ label: 'By Alert type', docID: 'helpAlertsDeleteType' }, - date: { - label: 'By date (older than)', + startdate: { + label: 'By date (start date)', + docID: 'helpAlertsDeleteDate', + isDatepicker: true + }, + enddate: { + label: 'By date (end than)', docID: 'helpAlertsDeleteDate', isDatepicker: true } @@ -440,9 +465,14 @@ type: args.data.type }); - if (args.data.date != "") + if (args.data.startdate != "") $.extend(data, { - olderthan: args.data.date + startdate: args.data.startdate + }); + + if (args.data.enddate != "") + $.extend(data, { + enddate: args.data.enddate }); $.ajax({ @@ -481,11 +511,17 @@ label: 'By Alert type', docID: 'helpAlertsArchiveType' }, - date: { - label: 'By date (older than)', + startdate: { + label: 'By date (start date)', + docID: 'helpAlertsArchiveDate', + isDatepicker: true + }, + enddate: { + label: 'By date (end date)', docID: 'helpAlertsArchiveDate', isDatepicker: true } + } }, action: function(args) { @@ -496,9 +532,14 @@ type: args.data.type }); - if (args.data.date != "") + if (args.data.startdate != "") $.extend(data, { - olderthan: args.data.date + startdate: args.data.startdate + }); + + if (args.data.enddate != "") + $.extend(data, { + enddate: args.data.enddate }); $.ajax({ From b9c15a483550de4946bb65cec21c35d0948a0e1b Mon Sep 17 00:00:00 2001 From: Prasanna Santhanam Date: Fri, 2 Aug 2013 22:15:23 +0530 Subject: [PATCH 096/763] isportable defaults to False Signed-off-by: Prasanna Santhanam (cherry picked from commit d993dda55a69e38bd0fba24166ad6fdf0e97cfc7) --- tools/marvin/marvin/integration/lib/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/marvin/marvin/integration/lib/base.py b/tools/marvin/marvin/integration/lib/base.py index e4cae7fec56..770ac3de5de 100755 --- a/tools/marvin/marvin/integration/lib/base.py +++ b/tools/marvin/marvin/integration/lib/base.py @@ -1088,7 +1088,7 @@ class PublicIPAddress: @classmethod def create(cls, apiclient, accountid=None, zoneid=None, domainid=None, services=None, - networkid=None, projectid=None, vpcid=None, isportable=None): + networkid=None, projectid=None, vpcid=None, isportable=False): """Associate Public IP address""" cmd = associateIpAddress.associateIpAddressCmd() From b4b280f9cf743e9104fa090ef7dc81f0fcf54b83 Mon Sep 17 00:00:00 2001 From: Prasanna Santhanam Date: Fri, 2 Aug 2013 22:22:50 +0530 Subject: [PATCH 097/763] moving rvr deployment planning to maint suites this suite performs enable/disable of pods/clusters in the zone causing other tests to fail deploying VMs. Move it to sequential execution suite in the maintenance tests Signed-off-by: Prasanna Santhanam --- .../{ => maint}/test_redundant_router_deployment_planning.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename test/integration/component/{ => maint}/test_redundant_router_deployment_planning.py (99%) diff --git a/test/integration/component/test_redundant_router_deployment_planning.py b/test/integration/component/maint/test_redundant_router_deployment_planning.py similarity index 99% rename from test/integration/component/test_redundant_router_deployment_planning.py rename to test/integration/component/maint/test_redundant_router_deployment_planning.py index 75fc3abed96..f4a6fb76d6c 100644 --- a/test/integration/component/test_redundant_router_deployment_planning.py +++ b/test/integration/component/maint/test_redundant_router_deployment_planning.py @@ -5,9 +5,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY From 922f902b3d2fa79cd03097c7c859f8762874f2f7 Mon Sep 17 00:00:00 2001 From: Alena Prokharchyk Date: Fri, 2 Aug 2013 10:53:34 -0700 Subject: [PATCH 098/763] CLOUDSTACK-4049: insert iso_id field to volumes table before altering the volumes view --- setup/db/db/schema-410to420.sql | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/setup/db/db/schema-410to420.sql b/setup/db/db/schema-410to420.sql index aeacff3928a..3ea63198d53 100644 --- a/setup/db/db/schema-410to420.sql +++ b/setup/db/db/schema-410to420.sql @@ -22,6 +22,10 @@ -- Disable foreign key checking SET foreign_key_checks = 0; + +ALTER TABLE volumes ADD COLUMN vm_snapshot_chain_size bigint(20) unsigned; +ALTER TABLE volumes ADD COLUMN iso_id bigint(20) unsigned; + -- All new inserts to the hypervisor_capabilities table should be after this ALTER TABLE `cloud`.`hypervisor_capabilities` ADD CONSTRAINT `uc_hypervisor` UNIQUE (`hypervisor_type`, `hypervisor_version`); @@ -81,6 +85,7 @@ CREATE TABLE `cloud`.`object_datastore_ref` ( PRIMARY KEY (`id`) ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8; + -- CREATE TABLE `cloud`.`data_store_provider` ( -- `id` bigint unsigned NOT NULL AUTO_INCREMENT COMMENT 'id', -- `name` varchar(255) NOT NULL COMMENT 'name of primary data store provider', @@ -2208,8 +2213,6 @@ CREATE TABLE `cloud_usage`.`usage_vmsnapshot` ( INDEX `i_usage_vmsnapshot` (`account_id`,`id`,`vm_id`,`created`) ) ENGINE=InnoDB CHARSET=utf8; -ALTER TABLE volumes ADD COLUMN vm_snapshot_chain_size bigint(20) unsigned; -ALTER TABLE volumes ADD COLUMN iso_id bigint(20) unsigned; INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Advanced', 'DEFAULT', 'management-server', 'kvm.ssh.to.agent', 'true', 'Specify whether or not the management server is allowed to SSH into KVM Agents'); From 8fe4c240bf25063491c44c55015a24bd19f831fd Mon Sep 17 00:00:00 2001 From: Alena Prokharchyk Date: Fri, 2 Aug 2013 13:15:07 -0700 Subject: [PATCH 099/763] CLOUDSTACK-4023: lock the network during the shutdownNetwork (just the way we do when implement network) --- .../com/cloud/network/NetworkManagerImpl.java | 123 ++++++++++-------- .../com/cloud/network/NetworkServiceImpl.java | 20 +-- .../VirtualNetworkApplianceManagerImpl.java | 58 ++++----- 3 files changed, 101 insertions(+), 100 deletions(-) diff --git a/server/src/com/cloud/network/NetworkManagerImpl.java b/server/src/com/cloud/network/NetworkManagerImpl.java index b5a1ca77e66..effee96c55d 100755 --- a/server/src/com/cloud/network/NetworkManagerImpl.java +++ b/server/src/com/cloud/network/NetworkManagerImpl.java @@ -2756,68 +2756,87 @@ public class NetworkManagerImpl extends ManagerBase implements NetworkManager, L @DB public boolean shutdownNetwork(long networkId, ReservationContext context, boolean cleanupElements) { boolean result = false; - Transaction txn = Transaction.currentTxn(); - txn.start(); - NetworkVO network = _networksDao.lockRow(networkId, true); - if (network == null) { - s_logger.debug("Unable to find network with id: " + networkId); - return false; - } - if (network.getState() != Network.State.Implemented && network.getState() != Network.State.Shutdown) { - s_logger.debug("Network is not implemented: " + network); - return false; - } + NetworkVO network = null; + try { + //do global lock for the network + network = _networksDao.acquireInLockTable(networkId, getNetworkLockTimeout()); + if (network == null) { + s_logger.warn("Unable to acquire lock for the network " + network + " as a part of network shutdown"); + return false; + } + if (s_logger.isDebugEnabled()) { + s_logger.debug("Lock is acquired for network " + network + " as a part of network shutdown"); + } + + if (network.getState() == Network.State.Allocated) { + s_logger.debug("Network is already shutdown: " + network); + return true; + } + + if (network.getState() != Network.State.Implemented && network.getState() != Network.State.Shutdown) { + s_logger.debug("Network is not implemented: " + network); + return false; + } - if (isSharedNetworkWithServices(network)) { - network.setState(Network.State.Shutdown); - _networksDao.update(network.getId(), network); - } else { - try { - stateTransitTo(network, Event.DestroyNetwork); - } catch (NoTransitionException e) { + if (isSharedNetworkWithServices(network)) { network.setState(Network.State.Shutdown); _networksDao.update(network.getId(), network); - } - } - txn.commit(); - - boolean success = shutdownNetworkElementsAndResources(context, cleanupElements, network); - - txn.start(); - if (success) { - if (s_logger.isDebugEnabled()) { - s_logger.debug("Network id=" + networkId + " is shutdown successfully, cleaning up corresponding resources now."); - } - NetworkGuru guru = AdapterBase.getAdapterByName(_networkGurus, network.getGuruName()); - NetworkProfile profile = convertNetworkToNetworkProfile(network.getId()); - guru.shutdown(profile, _networkOfferingDao.findById(network.getNetworkOfferingId())); - - applyProfileToNetwork(network, profile); - DataCenterVO zone = _dcDao.findById(network.getDataCenterId()); - if (isSharedNetworkOfferingWithServices(network.getNetworkOfferingId()) && (zone.getNetworkType() == NetworkType.Advanced)) { - network.setState(Network.State.Setup); } else { try { - stateTransitTo(network, Event.OperationSucceeded); + stateTransitTo(network, Event.DestroyNetwork); } catch (NoTransitionException e) { - network.setState(Network.State.Allocated); - network.setRestartRequired(false); + network.setState(Network.State.Shutdown); + _networksDao.update(network.getId(), network); } } - _networksDao.update(network.getId(), network); - _networksDao.clearCheckForGc(networkId); - result = true; - } else { - try { - stateTransitTo(network, Event.OperationFailed); - } catch (NoTransitionException e) { - network.setState(Network.State.Implemented); + + + boolean success = shutdownNetworkElementsAndResources(context, cleanupElements, network); + + Transaction txn = Transaction.currentTxn(); + txn.start(); + if (success) { + if (s_logger.isDebugEnabled()) { + s_logger.debug("Network id=" + networkId + " is shutdown successfully, cleaning up corresponding resources now."); + } + NetworkGuru guru = AdapterBase.getAdapterByName(_networkGurus, network.getGuruName()); + NetworkProfile profile = convertNetworkToNetworkProfile(network.getId()); + guru.shutdown(profile, _networkOfferingDao.findById(network.getNetworkOfferingId())); + + applyProfileToNetwork(network, profile); + DataCenterVO zone = _dcDao.findById(network.getDataCenterId()); + if (isSharedNetworkOfferingWithServices(network.getNetworkOfferingId()) && (zone.getNetworkType() == NetworkType.Advanced)) { + network.setState(Network.State.Setup); + } else { + try { + stateTransitTo(network, Event.OperationSucceeded); + } catch (NoTransitionException e) { + network.setState(Network.State.Allocated); + network.setRestartRequired(false); + } + } _networksDao.update(network.getId(), network); + _networksDao.clearCheckForGc(networkId); + result = true; + } else { + try { + stateTransitTo(network, Event.OperationFailed); + } catch (NoTransitionException e) { + network.setState(Network.State.Implemented); + _networksDao.update(network.getId(), network); + } + result = false; } - result = false; - } - txn.commit(); - return result; + txn.commit(); + return result; + } finally { + if (network != null) { + _networksDao.releaseFromLockTable(network.getId()); + if (s_logger.isDebugEnabled()) { + s_logger.debug("Lock is released for network " + network + " as a part of network shutdown"); + } + } + } } @Override diff --git a/server/src/com/cloud/network/NetworkServiceImpl.java b/server/src/com/cloud/network/NetworkServiceImpl.java index 90368be0a92..739f5cc9978 100755 --- a/server/src/com/cloud/network/NetworkServiceImpl.java +++ b/server/src/com/cloud/network/NetworkServiceImpl.java @@ -18,8 +18,6 @@ package com.cloud.network; import java.net.Inet6Address; import java.net.InetAddress; -import java.net.InetAddress; -import java.net.Inet6Address; import java.net.UnknownHostException; import java.security.InvalidParameterException; import java.sql.PreparedStatement; @@ -41,19 +39,11 @@ import javax.ejb.Local; import javax.inject.Inject; import javax.naming.ConfigurationException; -import com.cloud.network.vpc.dao.VpcDao; import org.apache.cloudstack.acl.ControlledEntity.ACLType; -import org.apache.cloudstack.acl.SecurityChecker; import org.apache.cloudstack.acl.SecurityChecker.AccessType; import org.apache.cloudstack.api.command.admin.network.DedicateGuestVlanRangeCmd; import org.apache.cloudstack.api.command.admin.network.ListDedicatedGuestVlanRangesCmd; import org.apache.cloudstack.api.command.admin.usage.ListTrafficTypeImplementorsCmd; -import org.apache.cloudstack.api.command.user.network.*; -import com.cloud.network.vpc.NetworkACL; -import com.cloud.network.vpc.dao.NetworkACLDao; -import org.apache.cloudstack.acl.ControlledEntity.ACLType; -import org.apache.cloudstack.acl.SecurityChecker.AccessType; -import org.apache.cloudstack.api.command.admin.usage.ListTrafficTypeImplementorsCmd; import org.apache.cloudstack.api.command.user.network.CreateNetworkCmd; import org.apache.cloudstack.api.command.user.network.ListNetworksCmd; import org.apache.cloudstack.api.command.user.network.RestartNetworkCmd; @@ -62,11 +52,6 @@ import org.apache.cloudstack.network.element.InternalLoadBalancerElementService; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; -import org.apache.log4j.Logger; -import org.springframework.stereotype.Component; -import org.apache.cloudstack.api.command.user.vm.ListNicsCmd; -import org.bouncycastle.util.IPAddress; - import com.cloud.api.ApiDBUtils; import com.cloud.configuration.Config; import com.cloud.configuration.ConfigurationManager; @@ -133,13 +118,15 @@ import com.cloud.network.element.VpcVirtualRouterElement; import com.cloud.network.guru.NetworkGuru; import com.cloud.network.rules.FirewallRule.Purpose; import com.cloud.network.rules.FirewallRuleVO; -import com.cloud.network.rules.PortForwardingRuleVO; import com.cloud.network.rules.RulesManager; import com.cloud.network.rules.dao.PortForwardingRulesDao; +import com.cloud.network.vpc.NetworkACL; import com.cloud.network.vpc.PrivateIpVO; import com.cloud.network.vpc.Vpc; import com.cloud.network.vpc.VpcManager; +import com.cloud.network.vpc.dao.NetworkACLDao; import com.cloud.network.vpc.dao.PrivateIpDao; +import com.cloud.network.vpc.dao.VpcDao; import com.cloud.offering.NetworkOffering; import com.cloud.offerings.NetworkOfferingVO; import com.cloud.offerings.dao.NetworkOfferingDao; @@ -160,7 +147,6 @@ import com.cloud.user.UserContext; import com.cloud.user.UserVO; import com.cloud.user.dao.AccountDao; import com.cloud.user.dao.UserDao; -import com.cloud.utils.AnnotationHelper; import com.cloud.utils.Journal; import com.cloud.utils.NumbersUtil; import com.cloud.utils.Pair; diff --git a/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java b/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java index 7a1a05b8201..a143c0c2b23 100755 --- a/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java +++ b/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java @@ -17,6 +17,33 @@ package com.cloud.network.router; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Collections; +import java.util.Comparator; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TimeZone; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import javax.ejb.Local; +import javax.inject.Inject; +import javax.naming.ConfigurationException; + +import org.apache.cloudstack.api.command.admin.router.UpgradeRouterCmd; +import org.apache.log4j.Logger; +import org.springframework.stereotype.Component; + import com.cloud.agent.AgentManager; import com.cloud.agent.AgentManager.OnError; import com.cloud.agent.Listener; @@ -146,8 +173,6 @@ import com.cloud.network.dao.Site2SiteVpnGatewayDao; import com.cloud.network.dao.UserIpv6AddressDao; import com.cloud.network.dao.VirtualRouterProviderDao; import com.cloud.network.dao.VpnUserDao; -import com.cloud.network.guru.NetworkGuru; -import com.cloud.network.guru.PublicNetworkGuru; import com.cloud.network.lb.LoadBalancingRule; import com.cloud.network.lb.LoadBalancingRule.LbDestination; import com.cloud.network.lb.LoadBalancingRule.LbHealthCheckPolicy; @@ -171,10 +196,8 @@ import com.cloud.offerings.NetworkOfferingVO; import com.cloud.offerings.dao.NetworkOfferingDao; import com.cloud.resource.ResourceManager; import com.cloud.server.ConfigurationServer; -import com.cloud.server.ManagementServer; import com.cloud.service.ServiceOfferingVO; import com.cloud.service.dao.ServiceOfferingDao; -import com.cloud.storage.GuestOSVO; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.Volume.Type; import com.cloud.storage.VolumeVO; @@ -196,7 +219,6 @@ import com.cloud.utils.NumbersUtil; import com.cloud.utils.Pair; import com.cloud.utils.PasswordGenerator; import com.cloud.utils.StringUtils; -import com.cloud.utils.component.AdapterBase; import com.cloud.utils.component.ManagerBase; import com.cloud.utils.concurrency.NamedThreadFactory; import com.cloud.utils.db.DB; @@ -233,32 +255,6 @@ import com.cloud.vm.dao.NicIpAliasVO; import com.cloud.vm.dao.UserVmDao; import com.cloud.vm.dao.UserVmDetailsDao; import com.cloud.vm.dao.VMInstanceDao; -import org.apache.cloudstack.api.command.admin.router.UpgradeRouterCmd; -import org.apache.log4j.Logger; -import org.springframework.stereotype.Component; - -import javax.ejb.Local; -import javax.inject.Inject; -import javax.naming.ConfigurationException; -import java.net.URI; -import java.util.ArrayList; -import java.util.Calendar; -import java.util.Collections; -import java.util.Comparator; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TimeZone; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; /** * VirtualNetworkApplianceManagerImpl manages the different types of virtual network appliances available in the Cloud Stack. From 838c972da06ed2c55ba03a2533fe8b4b9cfddbbf Mon Sep 17 00:00:00 2001 From: Min Chen Date: Thu, 1 Aug 2013 18:02:42 -0700 Subject: [PATCH 100/763] CLOUDSTACK-3748: AddS3Cmd NPE. --- .../api/command/admin/storage/AddS3Cmd.java | 28 ++++++++++++------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/api/src/org/apache/cloudstack/api/command/admin/storage/AddS3Cmd.java b/api/src/org/apache/cloudstack/api/command/admin/storage/AddS3Cmd.java index 3ad84fd5a51..70cde7892f4 100644 --- a/api/src/org/apache/cloudstack/api/command/admin/storage/AddS3Cmd.java +++ b/api/src/org/apache/cloudstack/api/command/admin/storage/AddS3Cmd.java @@ -91,8 +91,8 @@ public final class AddS3Cmd extends BaseCmd { @Override public void execute() throws ResourceUnavailableException, InsufficientCapacityException, - ServerApiException, ConcurrentOperationException, ResourceAllocationException, - NetworkRuleConflictException { + ServerApiException, ConcurrentOperationException, ResourceAllocationException, + NetworkRuleConflictException { AddImageStoreCmd cmd = new AddImageStoreCmd(); cmd.setProviderName("S3"); @@ -101,19 +101,27 @@ public final class AddS3Cmd extends BaseCmd { details.put(ApiConstants.S3_SECRET_KEY, this.getSecretKey()); details.put(ApiConstants.S3_END_POINT, this.getEndPoint()); details.put(ApiConstants.S3_BUCKET_NAME, this.getBucketName()); - details.put(ApiConstants.S3_HTTPS_FLAG, this.getHttpsFlag().toString()); - details.put(ApiConstants.S3_CONNECTION_TIMEOUT, this.getConnectionTimeout().toString()); - details.put(ApiConstants.S3_MAX_ERROR_RETRY, this.getMaxErrorRetry().toString()); - details.put(ApiConstants.S3_SOCKET_TIMEOUT, this.getSocketTimeout().toString()); + if (this.getHttpsFlag() != null) { + details.put(ApiConstants.S3_HTTPS_FLAG, this.getHttpsFlag().toString()); + } + if (this.getConnectionTimeout() != null) { + details.put(ApiConstants.S3_CONNECTION_TIMEOUT, this.getConnectionTimeout().toString()); + } + if (this.getMaxErrorRetry() != null) { + details.put(ApiConstants.S3_MAX_ERROR_RETRY, this.getMaxErrorRetry().toString()); + } + if (this.getSocketTimeout() != null) { + details.put(ApiConstants.S3_SOCKET_TIMEOUT, this.getSocketTimeout().toString()); + } try{ ImageStore result = _storageService.discoverImageStore(cmd); ImageStoreResponse storeResponse = null; if (result != null ) { - storeResponse = _responseGenerator.createImageStoreResponse(result); - storeResponse.setResponseName(getCommandName()); - storeResponse.setObjectName("secondarystorage"); - this.setResponseObject(storeResponse); + storeResponse = _responseGenerator.createImageStoreResponse(result); + storeResponse.setResponseName(getCommandName()); + storeResponse.setObjectName("secondarystorage"); + this.setResponseObject(storeResponse); } else { throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to add secondary storage"); } From ad20a9b6fa09d3595c2dec8c213cb3fee35d1973 Mon Sep 17 00:00:00 2001 From: Min Chen Date: Fri, 2 Aug 2013 11:51:56 -0700 Subject: [PATCH 101/763] CLOUDSTACK-3748:AddS3Cmd NPE. --- .../admin/storage/AddImageStoreCmd.java | 36 ++++++++-------- .../api/command/admin/storage/AddS3Cmd.java | 43 +++++++++++-------- 2 files changed, 42 insertions(+), 37 deletions(-) diff --git a/api/src/org/apache/cloudstack/api/command/admin/storage/AddImageStoreCmd.java b/api/src/org/apache/cloudstack/api/command/admin/storage/AddImageStoreCmd.java index 7e4409895c4..1552e0520a8 100644 --- a/api/src/org/apache/cloudstack/api/command/admin/storage/AddImageStoreCmd.java +++ b/api/src/org/apache/cloudstack/api/command/admin/storage/AddImageStoreCmd.java @@ -60,7 +60,7 @@ public class AddImageStoreCmd extends BaseCmd { @Parameter(name=ApiConstants.DETAILS, type=CommandType.MAP, description="the details for the image store. Example: details[0].key=accesskey&details[0].value=s389ddssaa&details[1].key=secretkey&details[1].value=8dshfsss") - private Map details; + private Map details; @@ -81,19 +81,19 @@ public class AddImageStoreCmd extends BaseCmd { return zoneId; } - public Map getDetails() { - Map detailsMap = null; - if (details != null && !details.isEmpty()) { - detailsMap = new HashMap(); - Collection props = details.values(); - Iterator iter = props.iterator(); - while (iter.hasNext()) { - HashMap detail = (HashMap) iter.next(); - String key = detail.get("key"); - String value = detail.get("value"); - detailsMap.put(key, value); - } - } + public Map getDetails() { + Map detailsMap = null; + if (details != null && !details.isEmpty()) { + detailsMap = new HashMap(); + Collection props = details.values(); + Iterator iter = props.iterator(); + while (iter.hasNext()) { + HashMap detail = (HashMap) iter.next(); + String key = detail.get("key"); + String value = detail.get("value"); + detailsMap.put(key, value); + } + } return detailsMap; } @@ -139,10 +139,10 @@ public class AddImageStoreCmd extends BaseCmd { ImageStore result = _storageService.discoverImageStore(this); ImageStoreResponse storeResponse = null; if (result != null ) { - storeResponse = _responseGenerator.createImageStoreResponse(result); - storeResponse.setResponseName(getCommandName()); - storeResponse.setObjectName("imagestore"); - this.setResponseObject(storeResponse); + storeResponse = _responseGenerator.createImageStoreResponse(result); + storeResponse.setResponseName(getCommandName()); + storeResponse.setObjectName("imagestore"); + this.setResponseObject(storeResponse); } else { throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to add secondary storage"); } diff --git a/api/src/org/apache/cloudstack/api/command/admin/storage/AddS3Cmd.java b/api/src/org/apache/cloudstack/api/command/admin/storage/AddS3Cmd.java index 70cde7892f4..0af1a85051f 100644 --- a/api/src/org/apache/cloudstack/api/command/admin/storage/AddS3Cmd.java +++ b/api/src/org/apache/cloudstack/api/command/admin/storage/AddS3Cmd.java @@ -94,25 +94,30 @@ public final class AddS3Cmd extends BaseCmd { ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { - AddImageStoreCmd cmd = new AddImageStoreCmd(); + AddImageStoreCmd cmd = new AddImageStoreCmd() { + @Override + public Map getDetails() { + Map dm = new HashMap(); + dm.put(ApiConstants.S3_ACCESS_KEY, getAccessKey()); + dm.put(ApiConstants.S3_SECRET_KEY, getSecretKey()); + dm.put(ApiConstants.S3_END_POINT, getEndPoint()); + dm.put(ApiConstants.S3_BUCKET_NAME, getBucketName()); + if (getHttpsFlag() != null) { + dm.put(ApiConstants.S3_HTTPS_FLAG, getHttpsFlag().toString()); + } + if (getConnectionTimeout() != null) { + dm.put(ApiConstants.S3_CONNECTION_TIMEOUT, getConnectionTimeout().toString()); + } + if (getMaxErrorRetry() != null) { + dm.put(ApiConstants.S3_MAX_ERROR_RETRY, getMaxErrorRetry().toString()); + } + if (getSocketTimeout() != null) { + dm.put(ApiConstants.S3_SOCKET_TIMEOUT, getSocketTimeout().toString()); + } + return dm; + } + }; cmd.setProviderName("S3"); - Map details = new HashMap(); - details.put(ApiConstants.S3_ACCESS_KEY, this.getAccessKey()); - details.put(ApiConstants.S3_SECRET_KEY, this.getSecretKey()); - details.put(ApiConstants.S3_END_POINT, this.getEndPoint()); - details.put(ApiConstants.S3_BUCKET_NAME, this.getBucketName()); - if (this.getHttpsFlag() != null) { - details.put(ApiConstants.S3_HTTPS_FLAG, this.getHttpsFlag().toString()); - } - if (this.getConnectionTimeout() != null) { - details.put(ApiConstants.S3_CONNECTION_TIMEOUT, this.getConnectionTimeout().toString()); - } - if (this.getMaxErrorRetry() != null) { - details.put(ApiConstants.S3_MAX_ERROR_RETRY, this.getMaxErrorRetry().toString()); - } - if (this.getSocketTimeout() != null) { - details.put(ApiConstants.S3_SOCKET_TIMEOUT, this.getSocketTimeout().toString()); - } try{ ImageStore result = _storageService.discoverImageStore(cmd); @@ -123,7 +128,7 @@ public final class AddS3Cmd extends BaseCmd { storeResponse.setObjectName("secondarystorage"); this.setResponseObject(storeResponse); } else { - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to add secondary storage"); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to add S3 secondary storage"); } } catch (DiscoveryException ex) { s_logger.warn("Exception: ", ex); From c3317ca5cfddc046bbbecb26178d2330de33e07b Mon Sep 17 00:00:00 2001 From: Sheng Yang Date: Fri, 2 Aug 2013 14:03:18 -0700 Subject: [PATCH 102/763] Automation: Fix test_vpc_network_staticnatrule.py --- .../test_vpc_network_staticnatrule.py | 46 +++++++++++++------ 1 file changed, 31 insertions(+), 15 deletions(-) diff --git a/test/integration/component/test_vpc_network_staticnatrule.py b/test/integration/component/test_vpc_network_staticnatrule.py index 539672e0b10..dd3d24908df 100644 --- a/test/integration/component/test_vpc_network_staticnatrule.py +++ b/test/integration/component/test_vpc_network_staticnatrule.py @@ -39,6 +39,7 @@ from marvin.integration.lib.common import (get_domain, cleanup_resources, list_routers) import socket +import time class Services: @@ -361,6 +362,21 @@ class TestVPCNetworkPFRules(cloudstackTestCase): self.fail("Failed to enable static NAT on IP: %s - %s" % ( public_ip.ipaddress.ipaddress, e)) + def delete_StaticNatRule_For_VM(self, vm, public_ip): + self.debug("Disabling static NAT for IP: %s" % + public_ip.ipaddress.ipaddress) + try: + StaticNATRule.disable( + self.apiclient, + ipaddressid=public_ip.ipaddress.id, + virtualmachineid=vm.id, + ) + self.debug("Static NAT disabled for IP: %s" % + public_ip.ipaddress.ipaddress) + except Exception as e: + self.fail("Failed to disabled static NAT on IP: %s - %s" % ( + public_ip.ipaddress.ipaddress, e)) + def acquire_Public_IP(self, network): self.debug("Associating public IP for network: %s" % network.name) public_ip = PublicIPAddress.create(self.apiclient, @@ -559,6 +575,10 @@ class TestVPCNetworkPFRules(cloudstackTestCase): vm_2 = self.create_VM_in_Network(network_2) public_ip_1 = self.acquire_Public_IP(network_1) public_ip_2 = self.acquire_Public_IP(network_2) + + # wait for VM to boot up + time.sleep(120) + router = self.stop_VPC_VRouter() self.create_StaticNatRule_For_VM(vm_1, public_ip_1, network_1) self.create_StaticNatRule_For_VM(vm_2, public_ip_2, network_2) @@ -594,10 +614,8 @@ class TestVPCNetworkPFRules(cloudstackTestCase): vm_2 = self.create_VM_in_Network(network_2) public_ip_1 = self.acquire_Public_IP(network_1) public_ip_2 = self.acquire_Public_IP(network_2) - router = self.stop_VPC_VRouter() self.create_StaticNatRule_For_VM(vm_1, public_ip_1, network_1) self.create_StaticNatRule_For_VM(vm_2, public_ip_2, network_2) - self.start_VPC_VRouter(router) self.check_ssh_into_vm(vm_1, public_ip_1, testnegative=False) self.check_ssh_into_vm(vm_2, public_ip_2, testnegative=False) return @@ -613,7 +631,7 @@ class TestVPCNetworkPFRules(cloudstackTestCase): # 2. Create a Network offering - NO1 with all supported services # 3. Add network1(10.1.1.1/24) using N01 to this VPC. # 4. Deploy vm1 in network1. - # 5. Use the Create PF rule for vm in network1. + # 5. Use the Create static nat rule for vm in network1. # 6. Successfully ssh into the Guest VM using the PF rule. # 7. Successfully wget a file on http server of VM1. # 8. Delete all PF rule @@ -626,12 +644,10 @@ class TestVPCNetworkPFRules(cloudstackTestCase): vm_1 = self.create_VM_in_Network(network_1) public_ip_1 = self.acquire_Public_IP(network_1) - nat_rule = self.create_StaticNatRule_For_VM(vm_1, public_ip_1, network_1) - http_rule = self.create_StaticNatRule_For_VM(vm_1, public_ip_1, network_1, self.services["http_rule"]) + self.create_StaticNatRule_For_VM(vm_1, public_ip_1, network_1) self.check_ssh_into_vm(vm_1, public_ip_1, testnegative=False) self.check_wget_from_vm(vm_1, public_ip_1, testnegative=False) - http_rule.delete(self.apiclient) - nat_rule.delete(self.apiclient) + self.delete_StaticNatRule_For_VM(vm_1, public_ip_1) self.check_ssh_into_vm(vm_1, public_ip_1, testnegative=True) self.check_wget_from_vm(vm_1, public_ip_1, testnegative=True) return @@ -671,10 +687,10 @@ class TestVPCNetworkPFRules(cloudstackTestCase): public_ip_2 = self.acquire_Public_IP(network_1) public_ip_3 = self.acquire_Public_IP(network_2) public_ip_4 = self.acquire_Public_IP(network_2) - nat_rule1 = self.create_StaticNatRule_For_VM(vm_1, public_ip_1, network_1) - nat_rule2 = self.create_StaticNatRule_For_VM(vm_2, public_ip_2, network_1) - nat_rule3 = self.create_StaticNatRule_For_VM(vm_3, public_ip_3, network_2) - nat_rule4 = self.create_StaticNatRule_For_VM(vm_4, public_ip_4, network_2) + self.create_StaticNatRule_For_VM(vm_1, public_ip_1, network_1) + self.create_StaticNatRule_For_VM(vm_2, public_ip_2, network_1) + self.create_StaticNatRule_For_VM(vm_3, public_ip_3, network_2) + self.create_StaticNatRule_For_VM(vm_4, public_ip_4, network_2) self.check_ssh_into_vm(vm_1, public_ip_1, testnegative=False) self.check_ssh_into_vm(vm_2, public_ip_2, testnegative=False) self.check_ssh_into_vm(vm_3, public_ip_1, testnegative=False) @@ -683,10 +699,10 @@ class TestVPCNetworkPFRules(cloudstackTestCase): self.check_wget_from_vm(vm_2, public_ip_2, testnegative=False) self.check_wget_from_vm(vm_3, public_ip_1, testnegative=False) self.check_wget_from_vm(vm_4, public_ip_2, testnegative=False) - nat_rule1.delete(self.apiclient) - nat_rule2.delete(self.apiclient) - nat_rule3.delete(self.apiclient) - nat_rule4.delete(self.apiclient) + self.delete_StaticNatRule_For_VM(vm_1, public_ip_1) + self.delete_StaticNatRule_For_VM(vm_2, public_ip_2) + self.delete_StaticNatRule_For_VM(vm_3, public_ip_3) + self.delete_StaticNatRule_For_VM(vm_4, public_ip_4) self.check_ssh_into_vm(vm_1, public_ip_1, testnegative=True) self.check_ssh_into_vm(vm_2, public_ip_2, testnegative=True) self.check_ssh_into_vm(vm_3, public_ip_1, testnegative=True) From 612ac971bc2b8c7b4284c0491bebefec6ba3002c Mon Sep 17 00:00:00 2001 From: Min Chen Date: Fri, 2 Aug 2013 14:25:11 -0700 Subject: [PATCH 103/763] CLOUDSTACK-4053: Removal of AddSwiftCmd and ListSwiftsCmd APIs broke api back compatibility. --- .../api/command/admin/swift/AddSwiftCmd.java | 122 ++++++++++++++++++ .../command/admin/swift/ListSwiftsCmd.java | 70 ++++++++++ client/tomcatconf/commands.properties.in | 4 + 3 files changed, 196 insertions(+) create mode 100644 api/src/org/apache/cloudstack/api/command/admin/swift/AddSwiftCmd.java create mode 100644 api/src/org/apache/cloudstack/api/command/admin/swift/ListSwiftsCmd.java diff --git a/api/src/org/apache/cloudstack/api/command/admin/swift/AddSwiftCmd.java b/api/src/org/apache/cloudstack/api/command/admin/swift/AddSwiftCmd.java new file mode 100644 index 00000000000..ea22429f093 --- /dev/null +++ b/api/src/org/apache/cloudstack/api/command/admin/swift/AddSwiftCmd.java @@ -0,0 +1,122 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.admin.swift; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.command.admin.storage.AddImageStoreCmd; +import org.apache.cloudstack.api.response.ImageStoreResponse; +import org.apache.log4j.Logger; + +import com.cloud.exception.DiscoveryException; +import com.cloud.storage.ImageStore; +import com.cloud.user.Account; + +@APICommand(name = "addSwift", description = "Adds Swift.", responseObject = ImageStoreResponse.class, since="3.0.0") +public class AddSwiftCmd extends BaseCmd { + public static final Logger s_logger = Logger.getLogger(AddSwiftCmd.class.getName()); + private static final String s_name = "addswiftresponse"; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.URL, type = CommandType.STRING, required = true, description = "the URL for swift") + private String url; + + @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, description = "the account for swift") + private String account; + + @Parameter(name = ApiConstants.USERNAME, type = CommandType.STRING, description = "the username for swift") + private String username; + + @Parameter(name = ApiConstants.KEY, type = CommandType.STRING, description = " key for the user for swift") + private String key; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public String getUrl() { + return url; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + public String getAccount() { + return account; + } + + public String getUsername() { + return username; + } + + public String getKey() { + return key; + } + + @Override + public String getCommandName() { + return s_name; + } + + @Override + public long getEntityOwnerId() { + return Account.ACCOUNT_ID_SYSTEM; + } + + @Override + public void execute(){ + AddImageStoreCmd cmd = new AddImageStoreCmd() { + @Override + public Map getDetails() { + Map dm = new HashMap(); + dm.put(ApiConstants.ACCOUNT, getAccount()); + dm.put(ApiConstants.USERNAME, getUsername()); + dm.put(ApiConstants.KEY, getKey()); + return dm; + } + }; + cmd.setProviderName("Swift"); + cmd.setUrl(this.getUrl()); + + try{ + ImageStore result = _storageService.discoverImageStore(cmd); + ImageStoreResponse storeResponse = null; + if (result != null ) { + storeResponse = _responseGenerator.createImageStoreResponse(result); + storeResponse.setResponseName(getCommandName()); + storeResponse.setObjectName("secondarystorage"); + this.setResponseObject(storeResponse); + } else { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to add Swift secondary storage"); + } + } catch (DiscoveryException ex) { + s_logger.warn("Exception: ", ex); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, ex.getMessage()); + } + } +} diff --git a/api/src/org/apache/cloudstack/api/command/admin/swift/ListSwiftsCmd.java b/api/src/org/apache/cloudstack/api/command/admin/swift/ListSwiftsCmd.java new file mode 100644 index 00000000000..b0408f43792 --- /dev/null +++ b/api/src/org/apache/cloudstack/api/command/admin/swift/ListSwiftsCmd.java @@ -0,0 +1,70 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.admin.swift; + +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseListCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.command.admin.storage.ListImageStoresCmd; +import org.apache.cloudstack.api.response.ImageStoreResponse; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.log4j.Logger; + +import com.cloud.user.Account; + +@APICommand(name = "listSwifts", description = "List Swift.", responseObject = ImageStoreResponse.class, since="3.0.0") +public class ListSwiftsCmd extends BaseListCmd { + public static final Logger s_logger = Logger.getLogger(ListSwiftsCmd.class.getName()); + private static final String s_name = "listswiftsresponse"; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.ID, type = CommandType.LONG, description = "the id of the swift") + private Long id; + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + public Long getId() { + return id; + } + + + @Override + public String getCommandName() { + return s_name; + } + + @Override + public long getEntityOwnerId() { + return Account.ACCOUNT_ID_SYSTEM; + } + + @Override + public void execute(){ + + ListImageStoresCmd cmd = new ListImageStoresCmd(); + cmd.setProvider("Swift"); + ListResponse response = _queryService.searchForImageStores(cmd); + response.setResponseName(getCommandName()); + this.setResponseObject(response); + } +} diff --git a/client/tomcatconf/commands.properties.in b/client/tomcatconf/commands.properties.in index fe3e4d9aad4..47282af70a9 100644 --- a/client/tomcatconf/commands.properties.in +++ b/client/tomcatconf/commands.properties.in @@ -242,6 +242,10 @@ deleteAlerts=1 #### system capacity commands listCapacity=3 +#### swift commands +addSwift=1 +listSwifts=1 + #### s3 commands addS3=1 listS3s=1 From 6940d256c9b0e3161f5cda7838b614c8a84fb899 Mon Sep 17 00:00:00 2001 From: Jessica Wang Date: Fri, 2 Aug 2013 14:21:15 -0700 Subject: [PATCH 104/763] CLOUDSTACK-2569: UI > Infrastructure > zone > physical network > VNMC provider > fix a bug that queryAsyncJobResult API kept being called even after the async job was finished. --- .../vnmcNetworkProvider.js | 29 +++++-------------- 1 file changed, 7 insertions(+), 22 deletions(-) diff --git a/ui/modules/vnmcNetworkProvider/vnmcNetworkProvider.js b/ui/modules/vnmcNetworkProvider/vnmcNetworkProvider.js index 2e1e12e80a1..f1d22d193bc 100644 --- a/ui/modules/vnmcNetworkProvider/vnmcNetworkProvider.js +++ b/ui/modules/vnmcNetworkProvider/vnmcNetworkProvider.js @@ -376,15 +376,14 @@ jobid: jid }, success: function (json) { - var result = json.queryasyncjobresultresponse; + var result = json.queryasyncjobresultresponse; if (result.jobstatus == 0) { return; //Job has not completed } else { + clearInterval(enableVnmcProviderIntervalID); if (result.jobstatus == 1) { args.response.success({ - data: { - state: 'Enabled' - } + data: result.jobresult.networkserviceprovider }); } else if (result.jobstatus == 2) { args.response.error(_s(result.jobresult.errortext)); @@ -419,13 +418,7 @@ } }); } - }, - - notification: { - poll: function (args) { - args.complete(); - } - } + } }, disable: { @@ -475,16 +468,14 @@ jobid: jid }, success: function (json) { - var result = json.queryasyncjobresultresponse; + var result = json.queryasyncjobresultresponse; if (result.jobstatus == 0) { return; //Job has not completed } else { clearInterval(disableVnmcProviderIntervalID); if (result.jobstatus == 1) { args.response.success({ - data: { - state: 'Disabled' - } + data: result.jobresult.networkserviceprovider }); } else if (result.jobstatus == 2) { args.response.error(_s(result.jobresult.errortext)); @@ -519,13 +510,7 @@ } }); } - }, - - notification: { - poll: function (args) { - args.complete(); - } - } + } } }, tabs: { From 13d44fd7a6317c3a0ebab2f577afa27f435a7334 Mon Sep 17 00:00:00 2001 From: Min Chen Date: Fri, 2 Aug 2013 16:05:16 -0700 Subject: [PATCH 105/763] CLOUDSTACK-3861 and CLOUDSTACK-3862: Deprecate old S3/Swift classes. --- api/src/com/cloud/storage/S3.java | 50 ---- api/src/com/cloud/storage/Swift.java | 37 --- .../cloudstack/api/ResponseGenerator.java | 8 - .../cloudstack/api/response/S3Response.java | 218 ------------------ .../api/response/SwiftResponse.java | 86 ------- .../api/command/test/ScaleVMCmdTest.java | 5 +- client/tomcatconf/applicationContext.xml.in | 2 - engine/schema/src/com/cloud/storage/S3VO.java | 205 ---------------- .../schema/src/com/cloud/storage/SwiftVO.java | 113 --------- .../src/com/cloud/storage/VMTemplateS3VO.java | 194 ---------------- .../src/com/cloud/storage/dao/S3Dao.java | 29 --- .../src/com/cloud/storage/dao/S3DaoImpl.java | 51 ---- .../cloud/storage/dao/VMTemplateS3Dao.java | 36 --- .../storage/dao/VMTemplateS3DaoImpl.java | 96 -------- server/src/com/cloud/api/ApiDBUtils.java | 24 +- .../src/com/cloud/api/ApiResponseHelper.java | 63 ++--- .../ConfigurationManagerImpl.java | 3 - .../com/cloud/storage/VolumeManagerImpl.java | 5 - .../cloud/template/TemplateManagerImpl.java | 3 - .../ChildTestConfiguration.java | 2 - 20 files changed, 30 insertions(+), 1200 deletions(-) delete mode 100644 api/src/com/cloud/storage/S3.java delete mode 100644 api/src/com/cloud/storage/Swift.java delete mode 100644 api/src/org/apache/cloudstack/api/response/S3Response.java delete mode 100644 api/src/org/apache/cloudstack/api/response/SwiftResponse.java delete mode 100644 engine/schema/src/com/cloud/storage/S3VO.java delete mode 100644 engine/schema/src/com/cloud/storage/SwiftVO.java delete mode 100644 engine/schema/src/com/cloud/storage/VMTemplateS3VO.java delete mode 100644 engine/schema/src/com/cloud/storage/dao/S3Dao.java delete mode 100644 engine/schema/src/com/cloud/storage/dao/S3DaoImpl.java delete mode 100644 engine/schema/src/com/cloud/storage/dao/VMTemplateS3Dao.java delete mode 100644 engine/schema/src/com/cloud/storage/dao/VMTemplateS3DaoImpl.java diff --git a/api/src/com/cloud/storage/S3.java b/api/src/com/cloud/storage/S3.java deleted file mode 100644 index 0c58a902923..00000000000 --- a/api/src/com/cloud/storage/S3.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package com.cloud.storage; - -import java.util.Date; - -import org.apache.cloudstack.api.Identity; -import org.apache.cloudstack.api.InternalIdentity; - -import com.cloud.agent.api.to.S3TO; - -public interface S3 extends InternalIdentity, Identity { - - String getAccessKey(); - - String getSecretKey(); - - String getEndPoint(); - - String getBucketName(); - - Integer getHttpsFlag(); - - Integer getConnectionTimeout(); - - Integer getMaxErrorRetry(); - - Integer getSocketTimeout(); - - Date getCreated(); - - S3TO toS3TO(); - -} diff --git a/api/src/com/cloud/storage/Swift.java b/api/src/com/cloud/storage/Swift.java deleted file mode 100644 index 028e96e48b8..00000000000 --- a/api/src/com/cloud/storage/Swift.java +++ /dev/null @@ -1,37 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -package com.cloud.storage; - -import org.apache.cloudstack.api.InternalIdentity; - -import com.cloud.agent.api.to.SwiftTO; - -public interface Swift extends InternalIdentity { - public long getId(); - - public String getUuid(); - - public String getUrl(); - - public String getAccount(); - - public String getUserName(); - - public String getKey(); - - public SwiftTO toSwiftTO(); -} diff --git a/api/src/org/apache/cloudstack/api/ResponseGenerator.java b/api/src/org/apache/cloudstack/api/ResponseGenerator.java index d8d07cb56fb..3e7d4b53076 100644 --- a/api/src/org/apache/cloudstack/api/ResponseGenerator.java +++ b/api/src/org/apache/cloudstack/api/ResponseGenerator.java @@ -73,10 +73,8 @@ import com.cloud.projects.ProjectInvitation; import com.cloud.region.ha.GlobalLoadBalancerRule; import com.cloud.server.ResourceTag; import com.cloud.storage.GuestOS; -import com.cloud.storage.S3; import com.cloud.storage.Snapshot; import com.cloud.storage.StoragePool; -import com.cloud.storage.Swift; import com.cloud.storage.Volume; import com.cloud.storage.snapshot.SnapshotPolicy; import com.cloud.storage.snapshot.SnapshotSchedule; @@ -151,7 +149,6 @@ import org.apache.cloudstack.api.response.RemoteAccessVpnResponse; import org.apache.cloudstack.api.response.ResourceCountResponse; import org.apache.cloudstack.api.response.ResourceLimitResponse; import org.apache.cloudstack.api.response.ResourceTagResponse; -import org.apache.cloudstack.api.response.S3Response; import org.apache.cloudstack.api.response.SecurityGroupResponse; import org.apache.cloudstack.api.response.ServiceOfferingResponse; import org.apache.cloudstack.api.response.ServiceResponse; @@ -164,7 +161,6 @@ import org.apache.cloudstack.api.response.SnapshotScheduleResponse; import org.apache.cloudstack.api.response.StaticRouteResponse; import org.apache.cloudstack.api.response.StorageNetworkIpRangeResponse; import org.apache.cloudstack.api.response.StoragePoolResponse; -import org.apache.cloudstack.api.response.SwiftResponse; import org.apache.cloudstack.api.response.SystemVmInstanceResponse; import org.apache.cloudstack.api.response.SystemVmResponse; import org.apache.cloudstack.api.response.TemplatePermissionsResponse; @@ -349,10 +345,6 @@ public interface ResponseGenerator { SystemVmInstanceResponse createSystemVmInstanceResponse(VirtualMachine systemVM); - SwiftResponse createSwiftResponse(Swift swift); - - S3Response createS3Response(S3 result); - PhysicalNetworkResponse createPhysicalNetworkResponse(PhysicalNetwork result); ServiceResponse createNetworkServiceResponse(Service service); diff --git a/api/src/org/apache/cloudstack/api/response/S3Response.java b/api/src/org/apache/cloudstack/api/response/S3Response.java deleted file mode 100644 index 259a3088c1e..00000000000 --- a/api/src/org/apache/cloudstack/api/response/S3Response.java +++ /dev/null @@ -1,218 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.cloudstack.api.response; - -import static org.apache.cloudstack.api.ApiConstants.ID; -import static org.apache.cloudstack.api.ApiConstants.S3_ACCESS_KEY; -import static org.apache.cloudstack.api.ApiConstants.S3_BUCKET_NAME; -import static org.apache.cloudstack.api.ApiConstants.S3_CONNECTION_TIMEOUT; -import static org.apache.cloudstack.api.ApiConstants.S3_END_POINT; -import static org.apache.cloudstack.api.ApiConstants.S3_HTTPS_FLAG; -import static org.apache.cloudstack.api.ApiConstants.S3_MAX_ERROR_RETRY; -import static org.apache.cloudstack.api.ApiConstants.S3_SECRET_KEY; -import static org.apache.cloudstack.api.ApiConstants.S3_SOCKET_TIMEOUT; - -import org.apache.cloudstack.api.BaseResponse; - -import com.cloud.serializer.Param; -import com.google.gson.annotations.SerializedName; - -public class S3Response extends BaseResponse { - - @SerializedName(ID) - @Param(description = "The ID of the S3 configuration") - private String id; - - @SerializedName(S3_ACCESS_KEY) - @Param(description = "The S3 access key") - private String accessKey; - - @SerializedName(S3_SECRET_KEY) - @Param(description = "The S3 secret key") - private String secretKey; - - @SerializedName(S3_END_POINT) - @Param(description = "The S3 end point") - private String endPoint; - - @SerializedName(S3_BUCKET_NAME) - @Param(description = "The name of the template storage bucket") - private String bucketName; - - @SerializedName(S3_HTTPS_FLAG) - @Param(description = "Connect to S3 using HTTPS?") - private Integer httpsFlag; - - @SerializedName(S3_CONNECTION_TIMEOUT) - @Param(description = "The connection timeout (milliseconds)") - private Integer connectionTimeout; - - @SerializedName(S3_MAX_ERROR_RETRY) - @Param(description = "The maximum number of time to retry a connection on error.") - private Integer maxErrorRetry; - - @SerializedName(S3_SOCKET_TIMEOUT) - @Param(description = "The connection socket (milliseconds)") - private Integer socketTimeout; - - @Override - public boolean equals(final Object thatObject) { - - if (this == thatObject) { - return true; - } - - if (thatObject == null || this.getClass() != thatObject.getClass()) { - return false; - } - - final S3Response thatS3Response = (S3Response) thatObject; - - if (this.httpsFlag != null ? !this.httpsFlag.equals(thatS3Response.httpsFlag) : thatS3Response.httpsFlag != null) { - return false; - } - - if (this.accessKey != null ? !this.accessKey.equals(thatS3Response.accessKey) : thatS3Response.accessKey != null) { - return false; - } - - if (this.connectionTimeout != null ? !this.connectionTimeout.equals(thatS3Response.connectionTimeout) : thatS3Response.connectionTimeout != null) { - return false; - } - - if (this.endPoint != null ? !this.endPoint.equals(thatS3Response.endPoint) : thatS3Response.endPoint != null) { - return false; - } - - if (this.id != null ? !this.id.equals(thatS3Response.id) : thatS3Response.id != null) { - return false; - } - - if (this.maxErrorRetry != null ? !this.maxErrorRetry.equals(thatS3Response.maxErrorRetry) : thatS3Response.maxErrorRetry != null) { - return false; - } - - if (this.secretKey != null ? !this.secretKey.equals(thatS3Response.secretKey) : thatS3Response.secretKey != null) { - return false; - } - - if (this.socketTimeout != null ? !this.socketTimeout.equals(thatS3Response.socketTimeout) : thatS3Response.socketTimeout != null) { - return false; - } - - if (this.bucketName != null ? !this.bucketName.equals(thatS3Response.bucketName) : thatS3Response.bucketName != null) { - return false; - } - - return true; - - } - - @Override - public int hashCode() { - - int result = this.id != null ? this.id.hashCode() : 0; - result = 31 * result + (this.accessKey != null ? this.accessKey.hashCode() : 0); - result = 31 * result + (this.secretKey != null ? this.secretKey.hashCode() : 0); - result = 31 * result + (this.endPoint != null ? this.endPoint.hashCode() : 0); - result = 31 * result + (this.bucketName != null ? this.bucketName.hashCode() : 0); - result = 31 * result + (this.httpsFlag != null ? this.httpsFlag : 0); - result = 31 * result + (this.connectionTimeout != null ? this.connectionTimeout.hashCode() : 0); - result = 31 * result + (this.maxErrorRetry != null ? this.maxErrorRetry.hashCode() : 0); - result = 31 * result + (this.socketTimeout != null ? this.socketTimeout.hashCode() : 0); - - return result; - - } - - @Override - public String getObjectId() { - return this.id; - } - - public void setObjectId(String id) { - this.id = id; - } - - public String getAccessKey() { - return this.accessKey; - } - - public void setAccessKey(final String accessKey) { - this.accessKey = accessKey; - } - - public String getSecretKey() { - return this.secretKey; - } - - public void setSecretKey(final String secretKey) { - this.secretKey = secretKey; - } - - public String getEndPoint() { - return this.endPoint; - } - - public void setEndPoint(final String endPoint) { - this.endPoint = endPoint; - } - - - public String getTemplateBucketName() { - return this.bucketName; - } - - public void setTemplateBucketName(final String templateBucketName) { - this.bucketName = templateBucketName; - } - - public Integer getHttpsFlag() { - return this.httpsFlag; - } - - public void setHttpsFlag(final Integer httpsFlag) { - this.httpsFlag = httpsFlag; - } - - public Integer getConnectionTimeout() { - return this.connectionTimeout; - } - - public void setConnectionTimeout(final Integer connectionTimeout) { - this.connectionTimeout = connectionTimeout; - } - - public Integer getMaxErrorRetry() { - return this.maxErrorRetry; - } - - public void setMaxErrorRetry(final Integer maxErrorRetry) { - this.maxErrorRetry = maxErrorRetry; - } - - public Integer getSocketTimeout() { - return this.socketTimeout; - } - - public void setSocketTimeout(final Integer socketTimeout) { - this.socketTimeout = socketTimeout; - } - -} diff --git a/api/src/org/apache/cloudstack/api/response/SwiftResponse.java b/api/src/org/apache/cloudstack/api/response/SwiftResponse.java deleted file mode 100644 index 08b260943ef..00000000000 --- a/api/src/org/apache/cloudstack/api/response/SwiftResponse.java +++ /dev/null @@ -1,86 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -package org.apache.cloudstack.api.response; - -import java.util.Date; - -import org.apache.cloudstack.api.ApiConstants; -import org.apache.cloudstack.api.BaseResponse; - -import com.cloud.serializer.Param; -import com.google.gson.annotations.SerializedName; - -public class SwiftResponse extends BaseResponse { - @SerializedName(ApiConstants.ID) - @Param(description = "the ID of swift") - private String id; - - @SerializedName(ApiConstants.URL) - @Param(description = "url for swift") - private String url; - - @SerializedName(ApiConstants.CREATED) - @Param(description = "the date and time the host was created") - private Date created; - - @SerializedName(ApiConstants.ACCOUNT) - @Param(description = "the account for swift") - private String account; - - @SerializedName(ApiConstants.ACCOUNT) - @Param(description = "the username for swift") - private String username; - - - - public void setId(String id) { - this.id = id; - } - - public String getUrl() { - return url; - } - - public void setUrl(String url) { - this.url = url; - } - - public Date getCreated() { - return created; - } - - public void setCreated(Date created) { - this.created = created; - } - - public String getAccount() { - return account; - } - - public void setAccount(String account) { - this.account = account; - } - - public String getUsername() { - return username; - } - - public void setUsername(String username) { - this.username = username; - } - -} diff --git a/api/test/org/apache/cloudstack/api/command/test/ScaleVMCmdTest.java b/api/test/org/apache/cloudstack/api/command/test/ScaleVMCmdTest.java index bb022986e2d..1e71739fe10 100644 --- a/api/test/org/apache/cloudstack/api/command/test/ScaleVMCmdTest.java +++ b/api/test/org/apache/cloudstack/api/command/test/ScaleVMCmdTest.java @@ -24,16 +24,12 @@ import org.apache.cloudstack.api.ResponseGenerator; import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.command.user.vm.ScaleVMCmd; -import org.apache.cloudstack.api.response.SwiftResponse; import org.apache.cloudstack.api.response.UserVmResponse; import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import org.mockito.Mockito; -import static org.mockito.Matchers.anyInt; - - import java.util.LinkedList; import java.util.List; @@ -45,6 +41,7 @@ public class ScaleVMCmdTest extends TestCase{ @Rule public ExpectedException expectedException = ExpectedException.none(); + @Override @Before public void setUp() { diff --git a/client/tomcatconf/applicationContext.xml.in b/client/tomcatconf/applicationContext.xml.in index 223f3453c21..205d3334096 100644 --- a/client/tomcatconf/applicationContext.xml.in +++ b/client/tomcatconf/applicationContext.xml.in @@ -293,7 +293,6 @@ - @@ -354,7 +353,6 @@ - diff --git a/engine/schema/src/com/cloud/storage/S3VO.java b/engine/schema/src/com/cloud/storage/S3VO.java deleted file mode 100644 index e30da0cbc2d..00000000000 --- a/engine/schema/src/com/cloud/storage/S3VO.java +++ /dev/null @@ -1,205 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package com.cloud.storage; - -import java.util.Date; - -import javax.persistence.Column; -import javax.persistence.Entity; -import javax.persistence.GeneratedValue; -import javax.persistence.GenerationType; -import javax.persistence.Id; -import javax.persistence.Table; - -import com.cloud.agent.api.to.S3TO; -import com.cloud.utils.db.GenericDao; - -//TODO: this will be removed after object_store merge. -@Entity -@Table(name = "s3") -public class S3VO implements S3 { - - public static final String ID_COLUMN_NAME = "id"; - - @Id - @GeneratedValue(strategy = GenerationType.IDENTITY) - @Column(name = ID_COLUMN_NAME) - private long id; - - @Column(name = "uuid") - private String uuid; - - @Column(name = "access_key") - private String accessKey; - - @Column(name = "secret_key") - private String secretKey; - - @Column(name = "end_point") - private String endPoint; - - @Column(name = "bucket") - private String bucketName; - - @Column(name = "https") - private Integer httpsFlag; - - @Column(name = "connection_timeout") - private Integer connectionTimeout; - - @Column(name = "max_error_retry") - private Integer maxErrorRetry; - - @Column(name = "socket_timeout") - private Integer socketTimeout; - - @Column(name = GenericDao.CREATED_COLUMN) - private Date created; - - public S3VO() { - super(); - } - - public S3VO(final String uuid, final String accessKey, final String secretKey, final String endPoint, - final String bucketName, final Boolean httpsFlag, final Integer connectionTimeout, - final Integer maxErrorRetry, final Integer socketTimeout, final Date created) { - - super(); - - this.uuid = uuid; - this.accessKey = accessKey; - this.secretKey = secretKey; - this.endPoint = endPoint; - this.bucketName = bucketName; - - Integer value = null; - if (httpsFlag != null) { - value = httpsFlag == false ? 0 : 1; - } - this.httpsFlag = value; - - this.connectionTimeout = connectionTimeout; - this.maxErrorRetry = maxErrorRetry; - this.socketTimeout = socketTimeout; - this.created = created; - - } - - @Override - public S3TO toS3TO() { - - Boolean httpsFlag = null; - if (this.httpsFlag != null) { - httpsFlag = this.httpsFlag == 0 ? false : true; - } - - return new S3TO(this.id, this.uuid, this.accessKey, this.secretKey, this.endPoint, this.bucketName, httpsFlag, - this.connectionTimeout, this.maxErrorRetry, this.socketTimeout, this.created, false); - - } - - public long getId() { - return this.id; - } - - public void setId(final long id) { - this.id = id; - } - - public String getUuid() { - return this.uuid; - } - - public void setUuid(final String uuid) { - this.uuid = uuid; - } - - public String getAccessKey() { - return this.accessKey; - } - - public void setAccessKey(final String accessKey) { - this.accessKey = accessKey; - } - - public String getSecretKey() { - return this.secretKey; - } - - public void setSecretKey(final String secretKey) { - this.secretKey = secretKey; - } - - public String getEndPoint() { - return this.endPoint; - } - - public void setEndPoint(final String endPoint) { - this.endPoint = endPoint; - } - - public String getBucketName() { - return this.bucketName; - } - - public void setBucketName(final String bucketName) { - this.bucketName = bucketName; - } - - public Integer getHttpsFlag() { - return this.httpsFlag; - } - - public void setHttpsFlag(final Integer httpsFlag) { - this.httpsFlag = httpsFlag; - } - - public Integer getConnectionTimeout() { - return this.connectionTimeout; - } - - public void setConnectionTimeout(final int connectionTimeout) { - this.connectionTimeout = connectionTimeout; - } - - public Integer getMaxErrorRetry() { - return this.maxErrorRetry; - } - - public void setMaxErrorRetry(final int maxErrorRetry) { - this.maxErrorRetry = maxErrorRetry; - } - - public Integer getSocketTimeout() { - return this.socketTimeout; - } - - public void setSocketTimeout(final int socketTimeout) { - this.socketTimeout = socketTimeout; - } - - public Date getCreated() { - return this.created; - } - - public void setCreated(final Date created) { - this.created = created; - } - -} diff --git a/engine/schema/src/com/cloud/storage/SwiftVO.java b/engine/schema/src/com/cloud/storage/SwiftVO.java deleted file mode 100644 index 1389242d21c..00000000000 --- a/engine/schema/src/com/cloud/storage/SwiftVO.java +++ /dev/null @@ -1,113 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -package com.cloud.storage; - -import java.util.Date; -import java.util.UUID; - -import javax.persistence.Column; -import javax.persistence.Entity; -import javax.persistence.GeneratedValue; -import javax.persistence.GenerationType; -import javax.persistence.Id; -import javax.persistence.Table; - -import org.apache.cloudstack.api.InternalIdentity; - -import com.cloud.agent.api.to.SwiftTO; -import com.cloud.utils.db.GenericDao; - -@Entity -@Table(name = "swift") -public class SwiftVO implements Swift, InternalIdentity { - - @Id - @GeneratedValue(strategy = GenerationType.IDENTITY) - @Column(name = "id") - private long id; - - @Column(name = "url") - String url; - - @Column(name = "account") - String account; - - @Column(name = "username") - String userName; - - @Column(name = "key") - String key; - - @Column(name = "uuid") - String uuid = UUID.randomUUID().toString(); - - @Column(name = GenericDao.CREATED_COLUMN) - private Date created; - - public SwiftVO() { - } - - public SwiftVO(String url, String account, String userName, String key) { - this.url = url; - this.account = account; - this.userName = userName; - this.key = key; - } - - @Override - public long getId() { - return id; - } - - @Override - public String getUrl() { - return url; - } - - @Override - public String getAccount() { - return account; - } - - @Override - public String getUserName() { - return userName; - } - - @Override - public String getKey() { - return key; - } - - public Date getCreated() { - return created; - } - - @Override - public SwiftTO toSwiftTO() { - return null; - } - - @Override - public String getUuid() { - return this.uuid; - } - - public void setUuid(String uuid) { - this.uuid = uuid; - } -} diff --git a/engine/schema/src/com/cloud/storage/VMTemplateS3VO.java b/engine/schema/src/com/cloud/storage/VMTemplateS3VO.java deleted file mode 100644 index e106bf7d1a5..00000000000 --- a/engine/schema/src/com/cloud/storage/VMTemplateS3VO.java +++ /dev/null @@ -1,194 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package com.cloud.storage; - -import com.cloud.utils.db.GenericDaoBase; -import org.apache.cloudstack.api.InternalIdentity; - -import javax.persistence.Column; -import javax.persistence.Entity; -import javax.persistence.GeneratedValue; -import javax.persistence.GenerationType; -import javax.persistence.Id; -import javax.persistence.Table; - -import java.text.DateFormat; -import java.util.Date; - -@Entity -@Table(name = "template_s3_ref") -public class VMTemplateS3VO implements InternalIdentity { - - public static final String S3_ID_COLUMN_NAME = "s3_id"; - - public static final String TEMPLATE_ID_COLUMN_NAME = "template_id"; - - @Id - @GeneratedValue(strategy = GenerationType.IDENTITY) - private long id; - - @Column(name = S3_ID_COLUMN_NAME) - private long s3Id; - - @Column(name = TEMPLATE_ID_COLUMN_NAME) - private long templateId; - - @Column(name = GenericDaoBase.CREATED_COLUMN) - private Date created; - - @Column(name = "size") - private Long size; - - @Column(name = "physical_size") - private Long physicalSize; - - public VMTemplateS3VO() { - super(); - } - - public VMTemplateS3VO(final long s3Id, final long templateId, final Date created, final Long size, - final Long physicalSize) { - - super(); - - this.s3Id = s3Id; - this.templateId = templateId; - this.created = created; - this.size = size; - this.physicalSize = physicalSize; - - } - - @Override - public boolean equals(final Object thatObject) { - - if (this == thatObject) { - return true; - } - - if (thatObject == null || getClass() != thatObject.getClass()) { - return false; - } - - final VMTemplateS3VO thatVMTemplateS3VO = (VMTemplateS3VO) thatObject; - - if (this.id != thatVMTemplateS3VO.id) { - return false; - } - - if (this.s3Id != thatVMTemplateS3VO.s3Id) { - return false; - } - - if (this.templateId != thatVMTemplateS3VO.templateId) { - return false; - } - - if (this.created != null ? !created.equals(thatVMTemplateS3VO.created) : thatVMTemplateS3VO.created != null) { - return false; - } - - if (this.physicalSize != null ? !physicalSize.equals(thatVMTemplateS3VO.physicalSize) - : thatVMTemplateS3VO.physicalSize != null) { - return false; - } - - if (this.size != null ? !size.equals(thatVMTemplateS3VO.size) : thatVMTemplateS3VO.size != null) { - return false; - } - - return true; - } - - @Override - public int hashCode() { - - int result = (int) (this.id ^ (this.id >>> 32)); - - result = 31 * result + (int) (this.s3Id ^ (this.s3Id >>> 32)); - result = 31 * result + (int) (this.templateId ^ (this.templateId >>> 32)); - result = 31 * result + (this.created != null ? this.created.hashCode() : 0); - result = 31 * result + (this.size != null ? this.size.hashCode() : 0); - result = 31 * result + (this.physicalSize != null ? this.physicalSize.hashCode() : 0); - - return result; - - } - - public long getId() { - return this.id; - } - - public void setId(final long id) { - this.id = id; - } - - public long getS3Id() { - return this.s3Id; - } - - public void setS3Id(final long s3Id) { - this.s3Id = s3Id; - } - - public long getTemplateId() { - return this.templateId; - } - - public void setTemplateId(final long templateId) { - this.templateId = templateId; - } - - public Date getCreated() { - return this.created; - } - - public void setCreated(final Date created) { - this.created = created; - } - - public Long getSize() { - return this.size; - } - - public void setSize(final Long size) { - this.size = size; - } - - public Long getPhysicalSize() { - return this.physicalSize; - } - - public void setPhysicalSize(final Long physicalSize) { - this.physicalSize = physicalSize; - } - - @Override - public String toString() { - - final StringBuilder stringBuilder = new StringBuilder("VMTemplateS3VO [ id: ").append(id).append(", created: ") - .append(DateFormat.getDateTimeInstance().format(created)).append(", physicalSize: ") - .append(physicalSize).append(", size: ").append(size).append(", templateId: ").append(templateId) - .append(", s3Id: ").append(s3Id).append(" ]"); - - return stringBuilder.toString(); - - } - -} diff --git a/engine/schema/src/com/cloud/storage/dao/S3Dao.java b/engine/schema/src/com/cloud/storage/dao/S3Dao.java deleted file mode 100644 index ebea3531339..00000000000 --- a/engine/schema/src/com/cloud/storage/dao/S3Dao.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package com.cloud.storage.dao; - -import com.cloud.agent.api.to.S3TO; -import com.cloud.storage.S3VO; -import com.cloud.utils.db.GenericDao; - -public interface S3Dao extends GenericDao { - - S3TO getS3TO(final Long id); - -} diff --git a/engine/schema/src/com/cloud/storage/dao/S3DaoImpl.java b/engine/schema/src/com/cloud/storage/dao/S3DaoImpl.java deleted file mode 100644 index 7316f018037..00000000000 --- a/engine/schema/src/com/cloud/storage/dao/S3DaoImpl.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package com.cloud.storage.dao; - -import com.cloud.agent.api.to.S3TO; -import com.cloud.storage.S3VO; -import com.cloud.utils.db.GenericDaoBase; - -import javax.ejb.Local; - -import org.springframework.stereotype.Component; - -@Component -@Local(S3Dao.class) -public class S3DaoImpl extends GenericDaoBase implements S3Dao { - - @Override - public S3TO getS3TO(final Long id) { - - if (id != null) { - - final S3VO s3VO = findById(id); - if (s3VO != null) { - return s3VO.toS3TO(); - } - - } - - // NOTE: Excluded listAll / shuffle operation implemented in - // SwiftDaoImpl ... - - return null; - - } -} diff --git a/engine/schema/src/com/cloud/storage/dao/VMTemplateS3Dao.java b/engine/schema/src/com/cloud/storage/dao/VMTemplateS3Dao.java deleted file mode 100644 index d36fb3a2257..00000000000 --- a/engine/schema/src/com/cloud/storage/dao/VMTemplateS3Dao.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package com.cloud.storage.dao; - -import com.cloud.storage.VMTemplateS3VO; -import com.cloud.utils.db.GenericDao; - -import java.util.List; - -public interface VMTemplateS3Dao extends GenericDao { - - List listByS3Id(long id); - - VMTemplateS3VO findOneByTemplateId(long id); - - VMTemplateS3VO findOneByS3Template(long s3Id, long templateId); - - void expungeAllByTemplateId(long templateId); - -} diff --git a/engine/schema/src/com/cloud/storage/dao/VMTemplateS3DaoImpl.java b/engine/schema/src/com/cloud/storage/dao/VMTemplateS3DaoImpl.java deleted file mode 100644 index d49645d944a..00000000000 --- a/engine/schema/src/com/cloud/storage/dao/VMTemplateS3DaoImpl.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package com.cloud.storage.dao; - -import static com.cloud.utils.db.SearchCriteria.Op.*; -import static com.cloud.storage.VMTemplateS3VO.*; - -import com.cloud.storage.VMTemplateS3VO; -import com.cloud.utils.db.GenericDaoBase; -import com.cloud.utils.db.SearchBuilder; -import com.cloud.utils.db.SearchCriteria; - -import javax.ejb.Local; - -import org.springframework.stereotype.Component; - -import java.util.List; - -@Component -@Local(VMTemplateS3Dao.class) -public class VMTemplateS3DaoImpl extends GenericDaoBase implements VMTemplateS3Dao { - - private final SearchBuilder searchBuilder; - - public VMTemplateS3DaoImpl() { - - super(); - - this.searchBuilder = createSearchBuilder(); - this.searchBuilder.and(S3_ID_COLUMN_NAME, this.searchBuilder.entity().getS3Id(), EQ) - .and(TEMPLATE_ID_COLUMN_NAME, this.searchBuilder.entity().getTemplateId(), EQ).done(); - - } - - @Override - public List listByS3Id(final long s3id) { - - final SearchCriteria criteria = this.searchBuilder.create(); - - criteria.setParameters(S3_ID_COLUMN_NAME, s3id); - - return this.listBy(criteria); - - } - - @Override - public VMTemplateS3VO findOneByTemplateId(final long templateId) { - - final SearchCriteria criteria = this.searchBuilder.create(); - - criteria.setParameters(TEMPLATE_ID_COLUMN_NAME, templateId); - - return this.findOneBy(criteria); - - } - - @Override - public VMTemplateS3VO findOneByS3Template(final long s3Id, final long templateId) { - - final SearchCriteria criteria = this.searchBuilder.create(); - - criteria.setParameters(S3_ID_COLUMN_NAME, s3Id); - criteria.setParameters(TEMPLATE_ID_COLUMN_NAME, templateId); - - return this.findOneBy(criteria); - - } - - @Override - public void expungeAllByTemplateId(long templateId) { - - final SearchCriteria criteria = this.searchBuilder.create(); - - criteria.setParameters(TEMPLATE_ID_COLUMN_NAME, templateId); - - this.expunge(criteria); - - } - -} diff --git a/server/src/com/cloud/api/ApiDBUtils.java b/server/src/com/cloud/api/ApiDBUtils.java index 7180a06be19..2263c4d1db1 100755 --- a/server/src/com/cloud/api/ApiDBUtils.java +++ b/server/src/com/cloud/api/ApiDBUtils.java @@ -233,7 +233,6 @@ import com.cloud.storage.StorageManager; import com.cloud.storage.StoragePool; import com.cloud.storage.StorageStats; import com.cloud.storage.UploadVO; -import com.cloud.storage.VMTemplateS3VO; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.Volume; import com.cloud.storage.Volume.Type; @@ -247,7 +246,6 @@ import com.cloud.storage.dao.SnapshotPolicyDao; import com.cloud.storage.dao.UploadDao; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VMTemplateDetailsDao; -import com.cloud.storage.dao.VMTemplateS3Dao; import com.cloud.storage.dao.VolumeDao; import com.cloud.storage.snapshot.SnapshotPolicy; import com.cloud.template.TemplateManager; @@ -330,7 +328,6 @@ public class ApiDBUtils { static PrimaryDataStoreDao _storagePoolDao; static VMTemplateDao _templateDao; static VMTemplateDetailsDao _templateDetailsDao; - static VMTemplateS3Dao _templateS3Dao; static UploadDao _uploadDao; static UserDao _userDao; static UserStatisticsDao _userStatsDao; @@ -441,7 +438,6 @@ public class ApiDBUtils { @Inject private PrimaryDataStoreDao storagePoolDao; @Inject private VMTemplateDao templateDao; @Inject private VMTemplateDetailsDao templateDetailsDao; - @Inject private VMTemplateS3Dao templateS3Dao; @Inject private UploadDao uploadDao; @Inject private UserDao userDao; @Inject private UserStatisticsDao userStatsDao; @@ -551,7 +547,6 @@ public class ApiDBUtils { _storagePoolDao = storagePoolDao; _templateDao = templateDao; _templateDetailsDao = templateDetailsDao; - _templateS3Dao = templateS3Dao; _uploadDao = uploadDao; _userDao = userDao; _userStatsDao = userStatsDao; @@ -891,15 +886,13 @@ public class ApiDBUtils { VMTemplateVO template = _templateDao.findByIdIncludingRemoved(templateId); if(template != null) { Map details = _templateDetailsDao.findDetails(templateId); - if(details != null && !details.isEmpty()) + if(details != null && !details.isEmpty()) { template.setDetails(details); + } } return template; } - public static VMTemplateS3VO findTemplateS3Ref(long templateId) { - return _templateS3Dao.findOneByTemplateId(templateId); - } public static UploadVO findUploadById(Long id) { return _uploadDao.findById(id); @@ -1187,10 +1180,11 @@ public class ApiDBUtils { List vos = _asVmGroupPolicyMapDao.listByVmGroupId(vmGroupId); for (AutoScaleVmGroupPolicyMapVO vo : vos) { AutoScalePolicy autoScalePolicy = _asPolicyDao.findById(vo.getPolicyId()); - if(autoScalePolicy.getAction().equals("scaleup")) + if(autoScalePolicy.getAction().equals("scaleup")) { scaleUpPolicyIds.add(autoScalePolicy.getId()); - else + } else { scaleDownPolicyIds.add(autoScalePolicy.getId()); + } } } public static String getKeyPairName(String sshPublicKey) { @@ -1211,10 +1205,11 @@ public class ApiDBUtils { List vos = _asVmGroupPolicyMapDao.listByVmGroupId(vmGroupId); for (AutoScaleVmGroupPolicyMapVO vo : vos) { AutoScalePolicy autoScalePolicy = _asPolicyDao.findById(vo.getPolicyId()); - if(autoScalePolicy.getAction().equals("scaleup")) + if(autoScalePolicy.getAction().equals("scaleup")) { scaleUpPolicies.add(autoScalePolicy); - else + } else { scaleDownPolicies.add(autoScalePolicy); + } } } @@ -1279,8 +1274,9 @@ public class ApiDBUtils { } public static String findJobInstanceUuid(AsyncJob job){ - if ( job == null || job.getInstanceId() == null) + if ( job == null || job.getInstanceId() == null) { return null; + } String jobInstanceId = null; diff --git a/server/src/com/cloud/api/ApiResponseHelper.java b/server/src/com/cloud/api/ApiResponseHelper.java index 4832d71472f..ab50293ed31 100755 --- a/server/src/com/cloud/api/ApiResponseHelper.java +++ b/server/src/com/cloud/api/ApiResponseHelper.java @@ -131,12 +131,10 @@ import com.cloud.storage.DiskOfferingVO; import com.cloud.storage.GuestOS; import com.cloud.storage.GuestOSCategoryVO; import com.cloud.storage.ImageStore; -import com.cloud.storage.S3; import com.cloud.storage.Snapshot; import com.cloud.storage.SnapshotVO; import com.cloud.storage.Storage.StoragePoolType; import com.cloud.storage.StoragePool; -import com.cloud.storage.Swift; import com.cloud.storage.Upload; import com.cloud.storage.UploadVO; import com.cloud.storage.VMTemplateVO; @@ -235,7 +233,6 @@ import org.apache.cloudstack.api.response.RemoteAccessVpnResponse; import org.apache.cloudstack.api.response.ResourceCountResponse; import org.apache.cloudstack.api.response.ResourceLimitResponse; import org.apache.cloudstack.api.response.ResourceTagResponse; -import org.apache.cloudstack.api.response.S3Response; import org.apache.cloudstack.api.response.SecurityGroupResponse; import org.apache.cloudstack.api.response.SecurityGroupRuleResponse; import org.apache.cloudstack.api.response.ServiceOfferingResponse; @@ -249,7 +246,6 @@ import org.apache.cloudstack.api.response.SnapshotScheduleResponse; import org.apache.cloudstack.api.response.StaticRouteResponse; import org.apache.cloudstack.api.response.StorageNetworkIpRangeResponse; import org.apache.cloudstack.api.response.StoragePoolResponse; -import org.apache.cloudstack.api.response.SwiftResponse; import org.apache.cloudstack.api.response.SystemVmInstanceResponse; import org.apache.cloudstack.api.response.SystemVmResponse; import org.apache.cloudstack.api.response.TemplatePermissionsResponse; @@ -460,10 +456,12 @@ public class ApiResponseHelper implements ResponseGenerator { vmSnapshotResponse.setDescription(vmSnapshot.getDescription()); vmSnapshotResponse.setDisplayName(vmSnapshot.getDisplayName()); UserVm vm = ApiDBUtils.findUserVmById(vmSnapshot.getVmId()); - if (vm != null) + if (vm != null) { vmSnapshotResponse.setVirtualMachineid(vm.getUuid()); - if (vmSnapshot.getParent() != null) + } + if (vmSnapshot.getParent() != null) { vmSnapshotResponse.setParentName(ApiDBUtils.getVMSnapshotById(vmSnapshot.getParent()).getDisplayName()); + } vmSnapshotResponse.setCurrent(vmSnapshot.getCurrent()); vmSnapshotResponse.setType(vmSnapshot.getType().toString()); vmSnapshotResponse.setObjectName("vmsnapshot"); @@ -514,35 +512,6 @@ public class ApiResponseHelper implements ResponseGenerator { return listHosts.get(0); } - @Override - public SwiftResponse createSwiftResponse(Swift swift) { - SwiftResponse swiftResponse = new SwiftResponse(); - swiftResponse.setId(swift.getUuid()); - swiftResponse.setUrl(swift.getUrl()); - swiftResponse.setAccount(swift.getAccount()); - swiftResponse.setUsername(swift.getUserName()); - swiftResponse.setObjectName("swift"); - return swiftResponse; - } - - @Override - public S3Response createS3Response(final S3 result) { - - final S3Response response = new S3Response(); - - response.setAccessKey(result.getAccessKey()); - response.setConnectionTimeout(result.getConnectionTimeout()); - response.setEndPoint(result.getEndPoint()); - response.setHttpsFlag(result.getHttpsFlag()); - response.setMaxErrorRetry(result.getMaxErrorRetry()); - response.setObjectId(result.getUuid()); - response.setSecretKey(result.getSecretKey()); - response.setSocketTimeout(result.getSocketTimeout()); - response.setTemplateBucketName(result.getBucketName()); - - return response; - - } @Override public VlanIpRangeResponse createVlanIpRangeResponse(Vlan vlan) { @@ -1896,10 +1865,11 @@ public class ApiResponseHelper implements ResponseGenerator { // convert account to projectIds Project project = ApiDBUtils.findProjectByProjectAccountIdIncludingRemoved(account.getId()); - if (project.getUuid() != null && !project.getUuid().isEmpty()) + if (project.getUuid() != null && !project.getUuid().isEmpty()) { projectIds.add(project.getUuid()); - else + } else { projectIds.add(String.valueOf(project.getId())); + } } } @@ -2010,8 +1980,9 @@ public class ApiResponseHelper implements ResponseGenerator { } if (so != null) { ServiceOffering soffering = ApiDBUtils.findServiceOfferingById(so); - if (soffering != null) + if (soffering != null) { response.setServiceOfferingId(soffering.getUuid()); + } } if (offering.getGuestType() != null) { @@ -2197,7 +2168,7 @@ public class ApiResponseHelper implements ResponseGenerator { } // populate network offering information - NetworkOffering networkOffering = (NetworkOffering) ApiDBUtils.findNetworkOfferingById(network.getNetworkOfferingId()); + NetworkOffering networkOffering = ApiDBUtils.findNetworkOfferingById(network.getNetworkOfferingId()); if (networkOffering != null) { response.setNetworkOfferingId(networkOffering.getUuid()); response.setNetworkOfferingName(networkOffering.getName()); @@ -2723,8 +2694,9 @@ public class ApiResponseHelper implements ResponseGenerator { public LBStickinessResponse createLBStickinessPolicyResponse(List stickinessPolicies, LoadBalancer lb) { LBStickinessResponse spResponse = new LBStickinessResponse(); - if (lb == null) + if (lb == null) { return spResponse; + } spResponse.setlbRuleId(lb.getUuid()); Account account = ApiDBUtils.findAccountById(lb.getAccountId()); if (account != null) { @@ -2751,8 +2723,9 @@ public class ApiResponseHelper implements ResponseGenerator { public LBHealthCheckResponse createLBHealthCheckPolicyResponse(List healthcheckPolicies, LoadBalancer lb) { LBHealthCheckResponse hcResponse = new LBHealthCheckResponse(); - if (lb == null) + if (lb == null) { return hcResponse; + } hcResponse.setlbRuleId(lb.getUuid()); Account account = ApiDBUtils.findAccountById(lb.getAccountId()); if (account != null) { @@ -3403,8 +3376,9 @@ public class ApiResponseHelper implements ResponseGenerator { usageRecResponse.setVmName(vm.getInstanceName()); usageRecResponse.setUsageId(vm.getUuid()); usageRecResponse.setSize(usageRecord.getSize()); - if(usageRecord.getOfferingId() != null) - usageRecResponse.setOfferingId(usageRecord.getOfferingId().toString()); + if(usageRecord.getOfferingId() != null) { + usageRecResponse.setOfferingId(usageRecord.getOfferingId().toString()); + } } if (usageRecord.getRawUsage() != null) { @@ -3423,8 +3397,9 @@ public class ApiResponseHelper implements ResponseGenerator { } public String getDateStringInternal(Date inputDate) { - if (inputDate == null) + if (inputDate == null) { return null; + } TimeZone tz = _usageSvc.getUsageTimezone(); Calendar cal = Calendar.getInstance(tz); diff --git a/server/src/com/cloud/configuration/ConfigurationManagerImpl.java b/server/src/com/cloud/configuration/ConfigurationManagerImpl.java index 9a21df7b806..59a67422396 100755 --- a/server/src/com/cloud/configuration/ConfigurationManagerImpl.java +++ b/server/src/com/cloud/configuration/ConfigurationManagerImpl.java @@ -170,7 +170,6 @@ import com.cloud.service.dao.ServiceOfferingDao; import com.cloud.service.dao.ServiceOfferingDetailsDao; import com.cloud.storage.DiskOfferingVO; import com.cloud.storage.dao.DiskOfferingDao; -import com.cloud.storage.dao.S3Dao; import com.cloud.test.IPRangeConfig; import com.cloud.user.Account; import com.cloud.user.AccountDetailVO; @@ -220,8 +219,6 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati @Inject DomainDao _domainDao; @Inject - S3Dao _s3Dao; - @Inject ServiceOfferingDao _serviceOfferingDao; @Inject ServiceOfferingDetailsDao _serviceOfferingDetailsDao; diff --git a/server/src/com/cloud/storage/VolumeManagerImpl.java b/server/src/com/cloud/storage/VolumeManagerImpl.java index c4846668580..a55f2326736 100644 --- a/server/src/com/cloud/storage/VolumeManagerImpl.java +++ b/server/src/com/cloud/storage/VolumeManagerImpl.java @@ -36,8 +36,6 @@ import com.cloud.utils.EnumUtils; import com.cloud.utils.NumbersUtil; import com.cloud.utils.Pair; import com.cloud.utils.UriUtils; -import com.cloud.utils.db.SearchCriteria; -import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -141,7 +139,6 @@ import com.cloud.storage.dao.StoragePoolWorkDao; import com.cloud.storage.dao.UploadDao; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VMTemplatePoolDao; -import com.cloud.storage.dao.VMTemplateS3Dao; import com.cloud.storage.dao.VolumeDao; import com.cloud.storage.dao.VolumeDetailsDao; import com.cloud.storage.download.DownloadMonitor; @@ -234,8 +231,6 @@ public class VolumeManagerImpl extends ManagerBase implements VolumeManager { @Inject protected VMTemplatePoolDao _vmTemplatePoolDao = null; @Inject - protected VMTemplateS3Dao _vmTemplateS3Dao; - @Inject protected VMTemplateDao _vmTemplateDao = null; @Inject protected StoragePoolHostDao _poolHostDao = null; diff --git a/server/src/com/cloud/template/TemplateManagerImpl.java b/server/src/com/cloud/template/TemplateManagerImpl.java index efef80b4208..91917a087bf 100755 --- a/server/src/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/com/cloud/template/TemplateManagerImpl.java @@ -139,7 +139,6 @@ import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VMTemplateDetailsDao; import com.cloud.storage.dao.VMTemplateHostDao; import com.cloud.storage.dao.VMTemplatePoolDao; -import com.cloud.storage.dao.VMTemplateS3Dao; import com.cloud.storage.dao.VMTemplateZoneDao; import com.cloud.storage.dao.VolumeDao; import com.cloud.storage.download.DownloadMonitor; @@ -220,8 +219,6 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, @Inject SnapshotDao _snapshotDao; @Inject - VMTemplateS3Dao _vmS3TemplateDao; - @Inject ConfigurationDao _configDao; @Inject ClusterDao _clusterDao; diff --git a/server/test/org/apache/cloudstack/networkoffering/ChildTestConfiguration.java b/server/test/org/apache/cloudstack/networkoffering/ChildTestConfiguration.java index 27faa4dcf00..c569e8a03bc 100644 --- a/server/test/org/apache/cloudstack/networkoffering/ChildTestConfiguration.java +++ b/server/test/org/apache/cloudstack/networkoffering/ChildTestConfiguration.java @@ -97,7 +97,6 @@ import com.cloud.server.ManagementService; import com.cloud.service.dao.ServiceOfferingDaoImpl; import com.cloud.service.dao.ServiceOfferingDetailsDaoImpl; import com.cloud.storage.dao.DiskOfferingDaoImpl; -import com.cloud.storage.dao.S3DaoImpl; import com.cloud.storage.dao.SnapshotDaoImpl; import com.cloud.storage.dao.StoragePoolDetailsDaoImpl; import com.cloud.storage.dao.VolumeDaoImpl; @@ -148,7 +147,6 @@ import org.apache.cloudstack.region.PortableIpRangeDaoImpl; DcDetailsDaoImpl.class, NicSecondaryIpDaoImpl.class, UserIpv6AddressDaoImpl.class, - S3DaoImpl.class, UserDaoImpl.class, NicDaoImpl.class, NetworkDomainDaoImpl.class, From cf220dd6332132aeb7eda98665b219735aca422d Mon Sep 17 00:00:00 2001 From: Amogh Vasekar Date: Fri, 2 Aug 2013 15:39:45 -0700 Subject: [PATCH 106/763] CLOUDSTACK-3647: Remove duplicate row Reviewed-by: Alena Prokharchyk --- setup/db/db/schema-410to420.sql | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/setup/db/db/schema-410to420.sql b/setup/db/db/schema-410to420.sql index 3ea63198d53..2f0f25c49d4 100644 --- a/setup/db/db/schema-410to420.sql +++ b/setup/db/db/schema-410to420.sql @@ -289,6 +289,12 @@ INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name) VALU INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name) VALUES (167, UUID(), 6, 'Windows Server 2012 (64-bit)'); INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name) VALUES (168, UUID(), 6, 'Windows Server 8 (64-bit)'); +# clean up row added in 3.0.6. +UPDATE `cloud`.`guest_os_hypervisor` set guest_os_id = 166 where guest_os_id = 206; +UPDATE `cloud`.`vm_template` set guest_os_id = 166 where guest_os_id = 206; +UPDATE `cloud`.`vm_instance` set guest_os_id = 166 where guest_os_id = 206; +DELETE IGNORE FROM `cloud`.`guest_os` where id=206; + INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name) VALUES (169, UUID(), 10, 'Ubuntu 11.04 (32-bit)'); INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name) VALUES (170, UUID(), 10, 'Ubuntu 11.04 (64-bit)'); INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name) VALUES (171, UUID(), 1, 'CentOS 6.3 (32-bit)'); From 7522f811672f66bc0cc13a33f4f3737ef03f22af Mon Sep 17 00:00:00 2001 From: Sheng Yang Date: Fri, 2 Aug 2013 16:45:40 -0700 Subject: [PATCH 107/763] Automation: BVT: Fix test_guest_vlan_range.py due to updated API --- test/integration/smoke/test_guest_vlan_range.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/test/integration/smoke/test_guest_vlan_range.py b/test/integration/smoke/test_guest_vlan_range.py index a99ad99c57a..8ea4719211e 100644 --- a/test/integration/smoke/test_guest_vlan_range.py +++ b/test/integration/smoke/test_guest_vlan_range.py @@ -68,6 +68,11 @@ class TestDedicateGuestVlanRange(cloudstackTestCase): cls._cleanup = [ cls.account, ] + + phy_networks = PhysicalNetwork.list( + cls.api_client + ) + cls.existed_vlan = phy_networks[0].vlan return @classmethod @@ -80,7 +85,7 @@ class TestDedicateGuestVlanRange(cloudstackTestCase): removeGuestVlanRangeResponse = \ physical_network.update(cls.api_client, id=physical_network.id, - removevlan=cls.services["vlan"]) + vlan=cls.existed_vlan) cleanup_resources(cls.api_client, cls._cleanup) except Exception as e: raise Exception("Warning: Exception during cleanup : %s" % e) @@ -128,7 +133,10 @@ class TestDedicateGuestVlanRange(cloudstackTestCase): physical_network_response = list_physical_network_response[0] self.debug("Adding guest vlan range") - addGuestVlanRangeResponse = physical_network_response.update(self.apiclient, id=physical_network_response.id, vlan=self.services["vlan"]) + + new_vlan = self.existed_vlan + "," + self.services["vlan"] + addGuestVlanRangeResponse = physical_network_response.update(self.apiclient, + id=physical_network_response.id, vlan=new_vlan) self.debug("Dedicating guest vlan range"); dedicate_guest_vlan_range_response = PhysicalNetwork.dedicate( From a4e256b311519e0cb2dc448bbbcfbd703ffc8219 Mon Sep 17 00:00:00 2001 From: Min Chen Date: Fri, 2 Aug 2013 17:59:24 -0700 Subject: [PATCH 108/763] CLOUDSTACK-4040: [Object_store_refactor] System VMs are not spinning up if S3 storage configuration is not part of zone creation wizard. --- .../cloud/resource/ResourceManagerImpl.java | 118 +++++++++--------- 1 file changed, 60 insertions(+), 58 deletions(-) diff --git a/server/src/com/cloud/resource/ResourceManagerImpl.java b/server/src/com/cloud/resource/ResourceManagerImpl.java index 80037343a5d..e5441af707b 100755 --- a/server/src/com/cloud/resource/ResourceManagerImpl.java +++ b/server/src/com/cloud/resource/ResourceManagerImpl.java @@ -367,8 +367,8 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, // Check if the zone exists in the system DataCenterVO zone = _dcDao.findById(dcId); if (zone == null) { - InvalidParameterValueException ex = new InvalidParameterValueException( - "Can't find zone by the id specified"); + InvalidParameterValueException ex = new InvalidParameterValueException( + "Can't find zone by the id specified"); ex.addProxyObject(String.valueOf(dcId), "dcId"); throw ex; } @@ -392,8 +392,8 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, } // check if pod belongs to the zone if (!Long.valueOf(pod.getDataCenterId()).equals(dcId)) { - InvalidParameterValueException ex = new InvalidParameterValueException( - "Pod with specified id doesn't belong to the zone " + dcId); + InvalidParameterValueException ex = new InvalidParameterValueException( + "Pod with specified id doesn't belong to the zone " + dcId); ex.addProxyObject(pod.getUuid(), "podId"); ex.addProxyObject(zone.getUuid(), "dcId"); throw ex; @@ -564,8 +564,8 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, if (clusterId != null) { ClusterVO cluster = _clusterDao.findById(clusterId); if (cluster == null) { - InvalidParameterValueException ex = new InvalidParameterValueException( - "can not find cluster for specified clusterId"); + InvalidParameterValueException ex = new InvalidParameterValueException( + "can not find cluster for specified clusterId"); ex.addProxyObject(clusterId.toString(), "clusterId"); throw ex; } else { @@ -587,7 +587,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, @Override public List discoverHosts(AddSecondaryStorageCmd cmd) throws IllegalArgumentException, DiscoveryException, - InvalidParameterValueException { + InvalidParameterValueException { Long dcId = cmd.getZoneId(); String url = cmd.getUrl(); return discoverHostsFull(dcId, null, null, null, url, null, null, "SecondaryStorage", null, null, false); @@ -620,11 +620,11 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, } // check if pod belongs to the zone if (!Long.valueOf(pod.getDataCenterId()).equals(dcId)) { - InvalidParameterValueException ex = new InvalidParameterValueException( - "Pod with specified podId" - + podId - + " doesn't belong to the zone with specified zoneId" - + dcId); + InvalidParameterValueException ex = new InvalidParameterValueException( + "Pod with specified podId" + + podId + + " doesn't belong to the zone with specified zoneId" + + dcId); ex.addProxyObject(pod.getUuid(), "podId"); ex.addProxyObject(zone.getUuid(), "dcId"); throw ex; @@ -683,11 +683,11 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, } catch (Exception e) { cluster = _clusterDao.findBy(clusterName, podId); if (cluster == null) { - CloudRuntimeException ex = new CloudRuntimeException( - "Unable to create cluster " - + clusterName - + " in pod with specified podId and data center with specified dcID", - e); + CloudRuntimeException ex = new CloudRuntimeException( + "Unable to create cluster " + + clusterName + + " in pod with specified podId and data center with specified dcID", + e); ex.addProxyObject(pod.getUuid(), "podId"); ex.addProxyObject(zone.getUuid(), "dcId"); throw ex; @@ -959,11 +959,11 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, if (hypervisorType == HypervisorType.VMware && Boolean.parseBoolean(_configDao.getValue(Config.VmwareUseNexusVSwitch.toString()))) { _clusterVSMMapDao.removeByClusterId(cmd.getId()); } - // remove from dedicated resources - DedicatedResourceVO dr = _dedicatedDao.findByClusterId(cluster.getId()); - if (dr != null) { - _dedicatedDao.remove(dr.getId()); - } + // remove from dedicated resources + DedicatedResourceVO dr = _dedicatedDao.findByClusterId(cluster.getId()); + if (dr != null) { + _dedicatedDao.remove(dr.getId()); + } } txn.commit(); @@ -1427,8 +1427,8 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, Iterator> it = _resourceStateAdapters.entrySet().iterator(); Object result = null; while (it.hasNext()) { - Map.Entry item = it - .next(); + Map.Entry item = it + .next(); ResourceStateAdapter adapter = item.getValue(); String msg = new String("Dispatching resource state event " + event + " to " + item.getKey()); @@ -1671,7 +1671,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, private void markHostAsDisconnected(HostVO host, StartupCommand[] cmds) { if (host == null) { // in case host is null due to some errors, try - // reloading the host from db + // reloading the host from db if (cmds != null) { StartupCommand firstCmd = cmds[0]; host = findHostByGuid(firstCmd.getGuid()); @@ -1721,13 +1721,13 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, host = findHostByGuid(firstCmd.getGuidWithoutResource()); } if (host != null && host.getRemoved() == null) { // host already - // added, no - // need to add - // again + // added, no + // need to add + // again s_logger.debug("Found the host " + host.getId() + " by guid: " + firstCmd.getGuid() + ", old host reconnected as new"); hostExists = true; // ensures that host status is left - // unchanged in case of adding same one - // again + // unchanged in case of adding same one + // again return null; } } @@ -1793,13 +1793,13 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, host = findHostByGuid(firstCmd.getGuidWithoutResource()); } if (host != null && host.getRemoved() == null) { // host already - // added, no - // need to add - // again + // added, no + // need to add + // again s_logger.debug("Found the host " + host.getId() + " by guid: " + firstCmd.getGuid() + ", old host reconnected as new"); hostExists = true; // ensures that host status is left - // unchanged in case of adding same one - // again + // unchanged in case of adding same one + // again return null; } } @@ -1808,29 +1808,29 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, GlobalLock addHostLock = GlobalLock.getInternLock("AddHostLock"); try { if (addHostLock.lock(ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_COOPERATION)) { // to - // safely - // determine - // first - // host - // in - // cluster - // in - // multi-MS - // scenario + // safely + // determine + // first + // host + // in + // cluster + // in + // multi-MS + // scenario try { host = createHostVO(cmds, resource, details, hostTags, ResourceStateAdapter.Event.CREATE_HOST_VO_FOR_DIRECT_CONNECT); if (host != null) { deferAgentCreation = !isFirstHostInCluster(host); // if - // first - // host - // in - // cluster - // no - // need - // to - // defer - // agent - // creation + // first + // host + // in + // cluster + // no + // need + // to + // defer + // agent + // creation } } finally { addHostLock.unlock(); @@ -1842,8 +1842,8 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, if (host != null) { if (!deferAgentCreation) { // if first host in cluster then - // create agent otherwise defer it to - // scan task + // create agent otherwise defer it to + // scan task attache = _agentMgr.handleDirectConnectAgent(host, cmds, resource, forRebalance); host = _hostDao.findById(host.getId()); // reload } else { @@ -2380,7 +2380,9 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, sc.addAnd(sc.getEntity().getDataCenterId(), Op.EQ, zoneId); } if (hostId != null) { - sc.addAnd(sc.getEntity().getId(), Op.EQ, hostId); + // exclude the given host, since we want to check what hypervisor is already handled + // in adding this new host + sc.addAnd(sc.getEntity().getId(), Op.NEQ, hostId); } sc.addAnd(sc.getEntity().getType(), Op.EQ, Host.Type.Routing); List hosts = sc.list(); @@ -2499,7 +2501,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, sc.addAnd(sc.getEntity().getStatus(), Op.EQ, Status.Up); sc.addAnd(sc.getEntity().getResourceState(), Op.EQ, ResourceState.Enabled); return sc.list(); - } + } @Override @DB From 149f4ffcb4121a344a43f4a9091d2892417b39e3 Mon Sep 17 00:00:00 2001 From: Min Chen Date: Sat, 3 Aug 2013 21:38:25 -0700 Subject: [PATCH 109/763] CLOUDSTACK-4034: Following global configuration missing from upgraded environment. --- setup/db/db/schema-410to420.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup/db/db/schema-410to420.sql b/setup/db/db/schema-410to420.sql index 2f0f25c49d4..f3ce3368751 100644 --- a/setup/db/db/schema-410to420.sql +++ b/setup/db/db/schema-410to420.sql @@ -2235,3 +2235,5 @@ CREATE VIEW `cloud`.`account_vmstats_view` AS INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Advanced', 'DEFAULT', 'management-server', 'healthcheck.update.interval', '600', 'Time Interval to fetch the LB health check states (in sec)'); INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Snapshots', 'DEFAULT', 'SnapshotManager', 'KVM.snapshot.enabled', 'false', 'whether snapshot is enabled for KVM hosts'); +INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Advanced', 'DEFAULT', 'management-server', 'eip.use.multiple.netscalers', 'false', 'Should be set to true, if there will be multiple NetScaler devices providing EIP service in a zone'); +INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Snapshots', 'DEFAULT', 'SnapshotManager', 'snapshot.backup.rightafter', 'true', 'backup snapshot right after snapshot is taken'); From 3e203f63b03bd186203de1584c9bd2ef8ae01eff Mon Sep 17 00:00:00 2001 From: Radhika PC Date: Mon, 5 Aug 2013 11:49:00 +0530 Subject: [PATCH 110/763] CLOUDSTACK-3546 edits and additions --- docs/en-US/added-API-commands-4.2.xml | 327 +++++++++++++++--------- docs/en-US/changed-API-commands-4.2.xml | 15 +- 2 files changed, 206 insertions(+), 136 deletions(-) diff --git a/docs/en-US/added-API-commands-4.2.xml b/docs/en-US/added-API-commands-4.2.xml index ddf80b19f13..bf9594f852a 100644 --- a/docs/en-US/added-API-commands-4.2.xml +++ b/docs/en-US/added-API-commands-4.2.xml @@ -135,328 +135,405 @@ The request parameters are elastic ip id and region id. - createVMSnapshot (create a virtual machine snapshot) + createVMSnapshot + Creates a virtual machine snapshot. - deleteVMSnapshot (delete a virtual machine snapshot) + deleteVMSnapshot + Deletes a virtual machine snapshot. - listVMSnapshot (show a virtual machine snapshot) + listVMSnapshot + Shows a virtual machine snapshot. - revertToVMSnapshot (return a virtual machine to the state and data saved in a given - snapshot) + revertToVMSnapshot + Returns a virtual machine to the state and data saved in a given snapshot. - createLBHealthCheckPolicy (creates a new health check policy for a load balancer rule; - see ) + createLBHealthCheckPolicy + Creates a new health check policy for a load balancer rule. - deleteLBHealthCheckPolicy (deletes an existing health check policy from a load balancer - rule) + deleteLBHealthCheckPolicy + Deletes an existing health check policy from a load balancer rule. - listLBHealthCheckPolicies (displays the health check policy for a load balancer - rule) + listLBHealthCheckPolicies + Displays the health check policy for a load balancer rule. - createEgressFirewallRules (creates an egress firewall rule on the guest network; see - ) + createEgressFirewallRules + Creates an egress firewall rule on the guest network. - deleteEgressFirewallRules (deletes a egress firewall rule on the guest network.) + deleteEgressFirewallRules + Deletes a egress firewall rule on the guest network. - listEgressFirewallRules (lists the egress firewall rules configured for a guest - network.) + listEgressFirewallRules + Lists the egress firewall rules configured for a guest network. - resetSSHKeyForVirtualMachine (Resets the SSHkey for virtual machine.) + resetSSHKeyForVirtualMachine + Resets the SSHkey for virtual machine. - addBaremetalHost (Adds a new host. Technically, this API command was present in v3.0.6, - but its functionality was disabled. See ) + addBaremetalHost + Adds a new host. Technically, this API command was present in v3.0.6, but its + functionality was disabled. - addBaremetalDhcp (Adds a DHCP server for bare metal hosts) + addBaremetalDhcp + Adds a DHCP server for bare metal hosts. - addBaremetalPxePingServer (Adds a PXE PING server for bare metal hosts) + addBaremetalPxePingServer + Adds a PXE PING server for bare metal hosts. addBaremetalPxeKickStartServer (Adds a PXE server for bare metal hosts) - listBaremetalDhcp (Shows the DHCP servers currently defined for bare metal - hosts) + listBaremetalDhcp + Shows the DHCP servers currently defined for bare metal hosts. - listBaremetalPxePingServer (Shows the PXE PING servers currently defined for bare metal - hosts) + listBaremetalPxePingServer + Shows the PXE PING servers currently defined for bare metal hosts. - addNicToVirtualMachine (Adds a new NIC to the specified VM on a selected network; see - ) + addNicToVirtualMachine + Adds a new NIC to the specified VM on a selected network. - removeNicFromVirtualMachine (Removes the specified NIC from a selected VM.) + removeNicFromVirtualMachine + Removes the specified NIC from a selected VM. - updateDefaultNicForVirtualMachine (Updates the specified NIC to be the default one for a - selected VM.) + updateDefaultNicForVirtualMachine + Updates the specified NIC to be the default one for a selected VM. - addRegion (Registers a Region into another Region; see ) + addRegion + Registers a Region into another Region. - updateRegion (Updates Region details: ID, Name, Endpoint, User API Key, and User Secret - Key.) + updateRegion + Updates Region details: ID, Name, Endpoint, User API Key, and User Secret Key. - removeRegion (Removes a Region from current Region.) + removeRegion + Removes a Region from current Region. - listRegions (Get all the Regions. They can be filtered by using the ID or Name.) + listRegions + Get all the Regions. They can be filtered by using the ID or Name. - getUser (This API can only be used by the Admin. Get user account details by using the - API Key.) + getUser + This API can only be used by the Admin. Get user account details by using the API + Key. - getApiLimit (Show number of remaining APIs for the invoking user in current - window) + getApiLimit + Shows number of remaining APIs for the invoking user in current window. - resetApiLimit (For root admin, if account ID parameter is passed, it will reset count - for that particular account, otherwise it will reset all counters) + resetApiLimit + For root admin, if account ID parameter is passed, it will reset count for that + particular account, otherwise it will reset all counters. - lockAccount (Locks an account) + lockAccount + Locks an account. - lockUser (Locks a user account) + lockUser + Locks a user account. - scaleVirtualMachine (Scales the virtual machine to a new service offering.) + scaleVirtualMachine + Scales the virtual machine to a new service offering. - migrateVirtualMachineWithVolume (Attempts migrating VM with its volumes to a different - host.) + migrateVirtualMachineWithVolume + Attempts migrating VM with its volumes to a different host. - dedicatePublicIpRange (Dedicates a Public IP range to an account.) + dedicatePublicIpRange + Dedicates a Public IP range to an account. - releasePublicIpRange (Releases a Public IP range back to the system pool.) + releasePublicIpRange + Releases a Public IP range back to the system pool. - dedicateGuestVlanRange (Dedicates a guest VLAN range to an account.) + dedicateGuestVlanRange + Dedicates a guest VLAN range to an account. - releaseDedicatedGuestVlanRange (Releases a dedicated guest VLAN range to the system.) - + releaseDedicatedGuestVlanRange + Releases a dedicated guest VLAN range to the system. - listDedicatedGuestVlanRanges (Lists dedicated guest VLAN ranges.) + listDedicatedGuestVlanRanges + Lists dedicated guest VLAN ranges. - updatePortForwardingRule (Updates a port forwarding rule. Only the private port and the - VM can be updated.) + updatePortForwardingRule + Updates a port forwarding rule. Only the private port and the VM can be updated. - scaleSystemVm (Scale the service offering for a systemVM, console proxy, or secondary - storage. The system VM must be in Stopped state for this command to take effect.) + scaleSystemVm + Scales the service offering for a systemVM, console proxy, or secondary storage. The + system VM must be in Stopped state for this command to take effect. - listDeploymentPlanners (Lists all the deployment planners available.) + listDeploymentPlanners + Lists all the deployment planners available. - addS3 (Adds a Amazon Simple Storage Service instance.) + addS3 + Adds a Amazon Simple Storage Service instance. - listS3s (Lists all the Amazon Simple Storage Service instances.) + listS3s + Lists all the Amazon Simple Storage Service instances. - findHostsForMigration (Find hosts suitable for migrating a VM to.) + findHostsForMigration + Finds hosts suitable for migrating a VM to. - releaseHostReservation (Releases host reservation.) + releaseHostReservation + Releases host reservation. - resizeVolume (Resizes a volume.) + resizeVolume + Resizes a volume. - updateVolume (Updates the volume.) + updateVolume + Updates the volume. - listStorageProviders (Lists storage providers.) + listStorageProviders + Lists storage providers. - findStoragePoolsForMigration (Lists storage pools available for migrating a volume.) - + findStoragePoolsForMigration + Lists storage pools available for migrating a volume. - createEgressFirewallRule (Creates a egress firewall rule for a given network. ) + createEgressFirewallRule + Creates a egress firewall rule for a given network. - deleteEgressFirewallRule (Deletes an egress firewall rule.) + deleteEgressFirewallRule + Deletes an egress firewall rule. - listEgressFirewallRules (Lists all egress firewall rules for network.) + listEgressFirewallRules + Lists all egress firewall rules for network. - updateNetworkACLItem (Updates ACL item with specified ID.) + updateNetworkACLItem + Updates ACL item with specified ID. - createNetworkACLList (Creates a Network ACL for the given VPC.) + createNetworkACLList + Creates a Network ACL for the given VPC. - deleteNetworkACLList (Deletes a Network ACL.) + deleteNetworkACLList + Deletes a Network ACL. - replaceNetworkACLList (Replaces ACL associated with a Network or private gateway.) - + replaceNetworkACLList + Replaces ACL associated with a Network or private gateway. - listNetworkACLLists (Lists all network ACLs.) + listNetworkACLLists + Lists all network ACLs. - addResourceDetail (Adds detail for the Resource.) + addResourceDetail + Adds detail for the Resource. - removeResourceDetail (Removes detail for the Resource.) + removeResourceDetail + Removes details of the resource. - listResourceDetails (List resource details.) + listResourceDetails + Lists resource details. - addNiciraNvpDevice (Adds a Nicira NVP device.) + addNiciraNvpDevice + Adds a Nicira NVP device. - deleteNiciraNvpDevice (Deletes a Nicira NVP device.) + deleteNiciraNvpDevice + Deletes a Nicira NVP device. - listNiciraNvpDevices (Lists Nicira NVP devices.) + listNiciraNvpDevices + Lists Nicira NVP devices. - listNiciraNvpDeviceNetworks (Lists network that are using a Nicira NVP device.) + listNiciraNvpDeviceNetworks + Lists network that are using a Nicira NVP device. - addBigSwitchVnsDevice (Adds a BigSwitch VNS device.) + addBigSwitchVnsDevice + Adds a BigSwitch VNS device. - deleteBigSwitchVnsDevice (Deletes a BigSwitch VNS device.) + deleteBigSwitchVnsDevice + Deletes a BigSwitch VNS device. - listBigSwitchVnsDevices (Lists BigSwitch VNS devices.) + listBigSwitchVnsDevices + Lists BigSwitch VNS devices. - configureSimulator (Configures a simulator.) + configureSimulator + Configures a simulator. - listApis (Lists all the available APIs on the server, provided by the API Discovery - plugin.) + listApis + Lists all the available APIs on the server, provided by the API Discovery plugin. - getApiLimit (Get the API limit count for the caller.) + getApiLimit + Gets the API limit count for the caller. - resetApiLimit (Reset the API count.) + resetApiLimit + Resets the API count. - assignToGlobalLoadBalancerRule (Assign load balancer rule or list of load balancer rules - to a global load balancer rules.) + assignToGlobalLoadBalancerRule + Assigns load balancer rule or list of load balancer rules to a global load balancer + rules. - removeFromGlobalLoadBalancerRule (Removes a load balancer rule association with global - load balancer rule) + removeFromGlobalLoadBalancerRule + Removes a load balancer rule association with global load balancer rule. - listVMSnapshot (List virtual machine snapshot by conditions) + listVMSnapshot + Lists virtual machine snapshot by conditions. - createLoadBalancer (Creates a Load Balancer) + createLoadBalancer + Creates a load balancer. - listLoadBalancers (Lists Load Balancers) + listLoadBalancers + Lists load balancers. - deleteLoadBalancer (Deletes a load balancer) + deleteLoadBalancer + Deletes a load balancer. - configureInternalLoadBalancerElement (Configures an Internal Load Balancer element.) - + configureInternalLoadBalancerElement + Configures an Internal Load Balancer element. - createInternalLoadBalancerElement (Create an Internal Load Balancer element.) + createInternalLoadBalancerElement + Creates an Internal Load Balancer element. - listInternalLoadBalancerElements (Lists all available Internal Load Balancer elements.) - + listInternalLoadBalancerElements + Lists all available Internal Load Balancer elements. - createAffinityGroup (Creates an affinity or anti-affinity group.) + createAffinityGroup + Creates an affinity or anti-affinity group. - deleteAffinityGroup (Deletes an affinity group.) + deleteAffinityGroup + Deletes an affinity group. - listAffinityGroups (Lists all the affinity groups.) + listAffinityGroups + Lists all the affinity groups. - updateVMAffinityGroup (Updates the affinity or anti-affinity group associations of a VM. - The VM has to be stopped and restarted for the new properties to take effect.) + updateVMAffinityGroup + Updates the affinity or anti-affinity group associations of a VM. The VM has to be + stopped and restarted for the new properties to take effect. - listAffinityGroupTypes (Lists affinity group types available.) + listAffinityGroupTypes + Lists affinity group types available. - stopInternalLoadBalancerVM (Stops an Internal LB VM.) + stopInternalLoadBalancerVM + Stops an Internal LB VM. - startInternalLoadBalancerVM (Starts an existing Internal LB VM.) + startInternalLoadBalancerVM + Starts an existing Internal LB VM. - listInternalLoadBalancerVMs (List internal LB VMs.) + listInternalLoadBalancerVMs + Lists internal LB VMs. - listNetworkIsolationMethods (Lists supported methods of network isolation.) + listNetworkIsolationMethods + Lists supported methods of network isolation. - dedicateZone (Dedicates a zone.) + dedicateZone + Dedicates a zone. - dedicatePod (Dedicates a pod.) + dedicatePod + Dedicates a pod. - dedicateCluster (Dedicate an existing cluster.) + dedicateCluster + Dedicates an existing cluster. - dedicateHost (Dedicates a host.) + dedicateHost + Dedicates a host. - releaseDedicatedZone (Release dedication of zone.) + releaseDedicatedZone + Releases dedication of zone. - releaseDedicatedPod (Release dedication for the pod.) + releaseDedicatedPod + Releases dedication for the pod. - releaseDedicatedCluster (Release dedication for cluster.) + releaseDedicatedCluster + Releases dedication for cluster. - releaseDedicatedHost (Release dedication for host.) + releaseDedicatedHost + Releases dedication for host. - listDedicatedZones (List dedicated zones.) + listDedicatedZones + Lists dedicated zones. - listDedicatedPods (Lists dedicated pods.) + listDedicatedPods + Lists dedicated pods. - listDedicatedClusters (Lists dedicated clusters.) + listDedicatedClusters + Lists dedicated clusters. - listDedicatedHosts (Lists dedicated hosts.) + listDedicatedHosts + Lists dedicated hosts.
diff --git a/docs/en-US/changed-API-commands-4.2.xml b/docs/en-US/changed-API-commands-4.2.xml index b1008875a51..0401f7211c4 100644 --- a/docs/en-US/changed-API-commands-4.2.xml +++ b/docs/en-US/changed-API-commands-4.2.xml @@ -388,24 +388,21 @@ listCapabilities The following new response parameters are added: apilimitinterval and - apilimitmax. - See . + apilimitmax. createServiceOffering The following new request parameters are added: deploymentplanner (optional), isvolatile (optional), serviceofferingdetails (optional). isvolatie indicates whether the service offering includes Volatile VM capability, - which will discard the VM's root disk and create a new one on reboot. See . + which will discard the VM's root disk and create a new one on reboot. The following new response parameters are added: deploymentplanner, isvolatile restoreVirtualMachine - The following request parameter is added: templateID (optional). This is used - to point to the new template ID when the base image is updated. See . + The following request parameter is added: templateID (optional). This is used to point to the + new template ID when the base image is updated. The following response parameters are added: diskioread, diskiowrite, diskkbsread, diskkbswrite, displayvm, isdynamicallyscalable, affinitygroup @@ -564,7 +561,6 @@ The following request parameters are added: cpuovercommitratio (optional), guestvswitchtype (optional), guestvswitchtype (optional), memoryovercommitratio (optional), publicvswitchtype (optional), publicvswitchtype (optional) - See . The following request parameters are added: cpuovercommitratio, memoryovercommitratio @@ -573,7 +569,6 @@ updateCluster The following request parameters are added: cpuovercommitratio, ramovercommitratio - See . @@ -584,7 +579,6 @@ The following request parameters are added: hypervisor (optional), provider (optional), scope (optional) The following request parameters have been made mandatory: podid, clusterid - See . The following response parameter has been added: hypervisor, scope, suitableformigration @@ -592,7 +586,6 @@ listStoragePools The following request parameter is added: scope (optional) - See . The following response parameters are added: hypervisor, scope, suitableformigration From f4876686a61c9af841cb90ac4db3bb1db53d5aa9 Mon Sep 17 00:00:00 2001 From: Radhika PC Date: Mon, 5 Aug 2013 13:50:13 +0530 Subject: [PATCH 111/763] edit changes to limit resources --- docs/en-US/limit-accounts-domains.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en-US/limit-accounts-domains.xml b/docs/en-US/limit-accounts-domains.xml index a864ee27ef3..78a642b3a5a 100644 --- a/docs/en-US/limit-accounts-domains.xml +++ b/docs/en-US/limit-accounts-domains.xml @@ -164,7 +164,7 @@
- Per-Domain Limits + Limiting Resource Usage in a Domain &PRODUCT; allows the configuration of limits on a domain basis. With a domain limit in place, all users still have their account limits. They are additionally limited, as a group, to not exceed the resource limits set on their domain. Domain limits aggregate the usage of From dbd321b02295c08bf5171baced9ce665e307305c Mon Sep 17 00:00:00 2001 From: Prasanna Santhanam Date: Mon, 5 Aug 2013 14:06:11 +0530 Subject: [PATCH 112/763] Fix pep8 Signed-off-by: Prasanna Santhanam --- tools/marvin/marvin/marvinPlugin.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/marvin/marvin/marvinPlugin.py b/tools/marvin/marvin/marvinPlugin.py index 1c6eb849ef0..aded17cca55 100644 --- a/tools/marvin/marvin/marvinPlugin.py +++ b/tools/marvin/marvin/marvinPlugin.py @@ -23,6 +23,7 @@ from marvin.cloudstackTestCase import cloudstackTestCase from marvin import deployDataCenter from nose.plugins.base import Plugin + class MarvinPlugin(Plugin): """ Custom plugin for the cloudstackTestCases to be run using nose From e297ffa4578dbff58a1e650839742e296c2f6e60 Mon Sep 17 00:00:00 2001 From: Radhika PC Date: Mon, 5 Aug 2013 14:14:25 +0530 Subject: [PATCH 113/763] CLOUDSTACK-850 fixed review comments from sheng/community --- docs/en-US/pvlan.xml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/en-US/pvlan.xml b/docs/en-US/pvlan.xml index d569507f973..f647d932538 100644 --- a/docs/en-US/pvlan.xml +++ b/docs/en-US/pvlan.xml @@ -113,19 +113,19 @@ ports. Configure the switch port connected to the router in PVLAN promiscuous trunk mode, which would translate an isolated VLAN to primary VLAN for the PVLAN-unaware router. Note that only Cisco Catalyst 4500 has the PVLAN promiscuous trunk mode to connect - both normal VLAN and PVLAN to a PVLAN-unaware switch. For other Catalyst PVLAN support - switch, connect the switch to upper switch by using cables. The number of cables should be - greater than the number of PVLANs used. + both normal VLAN and PVLAN to a PVLAN-unaware switch. For the other Catalyst PVLAN support + switch, connect the switch to upper switch by using cables, one each for a PVLAN + pair. Configure private VLAN on your physical switches out-of-band. - Before you use PVLAN on XenServer and KVM, enable Open vSwitch (OVS) . + Before you use PVLAN on XenServer and KVM, enable Open vSwitch (OVS). - OVS on XenServer and KVM does not support PVLAN. Therefore, simulate PVLAN on OVS - for XenServer and KVM by modifying the flow table and tagging every traffic leaving - guest VMs with the secondary VLAN ID. + OVS on XenServer and KVM does not support PVLAN natively. Therefore, &PRODUCT; + managed to simulate PVLAN on OVS for XenServer and KVM by modifying the flow + table. @@ -176,8 +176,8 @@ VLAN ID: The unique ID of the VLAN. - Isolated VLAN ID: The unique ID of the Secondary - Isolated VLAN. + Secondary Isolated VLAN ID: The unique ID of the + Secondary Isolated VLAN. For the description on Secondary Isolated VLAN, see . From 465bfbadb3f919a42486babd0c724b78654d9c04 Mon Sep 17 00:00:00 2001 From: Radhika PC Date: Mon, 5 Aug 2013 15:00:14 +0530 Subject: [PATCH 114/763] CLOUDSTACK-817 likitha/community doc comments fixed --- docs/en-US/ip-vlan-tenant.xml | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/docs/en-US/ip-vlan-tenant.xml b/docs/en-US/ip-vlan-tenant.xml index f66fde1661f..31cbde27d42 100644 --- a/docs/en-US/ip-vlan-tenant.xml +++ b/docs/en-US/ip-vlan-tenant.xml @@ -19,19 +19,26 @@ under the License. -->
- Reserving Public IP Addresses and VLANs Per Account + Reserving Public IP Addresses and VLANs for Account &PRODUCT; provides you the ability to reserve a set of public IP addresses and VLANs - exclusively for an account. During zone creation, you can continue to define a set of VLANs and - multiple public IP ranges. This feature extends the functionality to enable you to dedicate a - fixed set of VLANs and guest IP addresses for a tenant. + exclusively for an account and domain. During zone creation, you can continue defining a set of + VLANs and multiple public IP ranges. This feature extends the functionality to enable you to + dedicate a fixed set of VLANs and guest IP addresses for a tenant. + Note that if an account has consumed all the VLANs and IPs dedicated to it, the account can + acquire two more resources from the system. &PRODUCT; provides the root admin with two + configuration parameter to modify this default behavior—use.system.public.ips and + use.system.guest.vlans. These global parameters enable the root admin to disallow an account + from acquiring public IPs and guest VLANs from the system, if the account has dedicated + resources and these dedicated resources have all been consumed. Both these configurations are + configurable at the account level. This feature provides you the following capabilities: Reserve a VLAN range and public IP address range from an Advanced zone and assign it to - a domain or account + an account or domain - Disassociate a VLAN and public IP address range from an domain or account + Disassociate a VLAN and public IP address range from an account or domain View the number of public IP addresses allocated to an account From ddf91226ea228632b86ef5be0a7029d246cb5e1b Mon Sep 17 00:00:00 2001 From: Nitin Mehta Date: Mon, 5 Aug 2013 17:12:52 +0530 Subject: [PATCH 115/763] CLOUDSTACK-3913 Private templates would now get copied to only one of image storage chosen randamly as was the case earlier. Dont throw an exception for uploading volumes when there are multiple image stores, instead choose one of them randomly Signed off by : nitin mehta --- .../cloudstack/storage/image/BaseImageStoreDriverImpl.java | 3 ++- .../src/com/cloud/template/HypervisorTemplateAdapter.java | 6 ++++++ server/src/com/cloud/template/TemplateManagerImpl.java | 7 +++---- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/engine/storage/src/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java b/engine/storage/src/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java index c8d4e4533fa..2905f081061 100644 --- a/engine/storage/src/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java +++ b/engine/storage/src/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java @@ -106,7 +106,8 @@ public abstract class BaseImageStoreDriverImpl implements ImageStoreDriver { } @Override - public void createAsync(DataStore dataStore, DataObject data, AsyncCompletionCallback callback) { + public void + createAsync(DataStore dataStore, DataObject data, AsyncCompletionCallback callback) { CreateContext context = new CreateContext(callback, data); AsyncCallbackDispatcher caller = AsyncCallbackDispatcher .create(this); diff --git a/server/src/com/cloud/template/HypervisorTemplateAdapter.java b/server/src/com/cloud/template/HypervisorTemplateAdapter.java index 21a0f6e559e..de0ed7e51bc 100755 --- a/server/src/com/cloud/template/HypervisorTemplateAdapter.java +++ b/server/src/com/cloud/template/HypervisorTemplateAdapter.java @@ -18,6 +18,7 @@ package com.cloud.template; import java.net.MalformedURLException; import java.net.URL; +import java.util.Collections; import java.util.List; import java.util.concurrent.ExecutionException; @@ -180,6 +181,8 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { if ( imageStores == null || imageStores.size() == 0 ){ throw new CloudRuntimeException("Unable to find image store to download template "+ profile.getTemplate()); } + + Collections.shuffle(imageStores);// For private templates choose a random store. TODO - Have a better algorithm based on size, no. of objects, load etc. for (DataStore imageStore : imageStores) { TemplateInfo tmpl = this.imageFactory.getTemplate(template.getId(), imageStore); CreateTemplateContext context = new CreateTemplateContext(null, tmpl); @@ -187,6 +190,9 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { caller.setCallback(caller.getTarget().createTemplateAsyncCallBack(null, null)); caller.setContext(context); this.imageService.createTemplateAsync(tmpl, imageStore, caller); + if( !(profile.getIsPublic() || profile.getFeatured()) ){ // If private template then break + break; + } } _resourceLimitMgr.incrementResourceCount(profile.getAccountId(), ResourceType.template); diff --git a/server/src/com/cloud/template/TemplateManagerImpl.java b/server/src/com/cloud/template/TemplateManagerImpl.java index 91917a087bf..d2b2afef3df 100755 --- a/server/src/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/com/cloud/template/TemplateManagerImpl.java @@ -349,11 +349,10 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, if (storeUuid != null) { imageStore = this._dataStoreMgr.getDataStore(storeUuid, DataStoreRole.Image); } else { - List stores = this._dataStoreMgr.getImageStoresByScope(new ZoneScope(zoneId)); - if (stores.size() > 1) { - throw new CloudRuntimeException("multiple image stores, don't know which one to use"); + imageStore = this._dataStoreMgr.getImageStore(zoneId); + if (imageStore == null) { + throw new CloudRuntimeException("cannot find an image store for zone " + zoneId); } - imageStore = stores.get(0); } return imageStore; From 40e7502bcb02a3301347f2debfb09a920306e796 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Mon, 5 Aug 2013 18:21:19 +0530 Subject: [PATCH 116/763] CLOUDSTACK-3228: system vms are not comming up in zone with two cluster xen and kvm CLOUDSTACK-3631: Enhance System vm deployment retry mechanism Signed off by : Nitin Mehta --- .../com/cloud/storage/dao/VMTemplateDao.java | 2 +- .../cloud/storage/dao/VMTemplateDaoImpl.java | 45 +++++++++++++++---- .../consoleproxy/ConsoleProxyManagerImpl.java | 25 +++++++---- .../SecondaryStorageManagerImpl.java | 21 ++++++--- 4 files changed, 69 insertions(+), 24 deletions(-) diff --git a/engine/schema/src/com/cloud/storage/dao/VMTemplateDao.java b/engine/schema/src/com/cloud/storage/dao/VMTemplateDao.java index c3d44bdb6aa..700ccf574b6 100755 --- a/engine/schema/src/com/cloud/storage/dao/VMTemplateDao.java +++ b/engine/schema/src/com/cloud/storage/dao/VMTemplateDao.java @@ -66,7 +66,7 @@ public interface VMTemplateDao extends GenericDao { VMTemplateVO findSystemVMTemplate(long zoneId); - VMTemplateVO findSystemVMTemplate(long zoneId, HypervisorType hType); + VMTemplateVO findSystemVMReadyTemplate(long zoneId, HypervisorType hypervisorType); VMTemplateVO findRoutingTemplate(HypervisorType type, String templateName); diff --git a/engine/schema/src/com/cloud/storage/dao/VMTemplateDaoImpl.java b/engine/schema/src/com/cloud/storage/dao/VMTemplateDaoImpl.java index 9e7599052a7..924322583f0 100755 --- a/engine/schema/src/com/cloud/storage/dao/VMTemplateDaoImpl.java +++ b/engine/schema/src/com/cloud/storage/dao/VMTemplateDaoImpl.java @@ -23,11 +23,15 @@ import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.Map; +import java.util.Collections; import javax.ejb.Local; import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.storage.VMTemplateStorageResourceAssoc; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -75,6 +79,8 @@ public class VMTemplateDaoImpl extends GenericDaoBase implem DomainDao _domainDao; @Inject DataCenterDao _dcDao; + @Inject + TemplateDataStoreDao _templateDataStoreDao; private static final String SELECT_S3_CANDIDATE_TEMPLATES = "SELECT t.id, t.unique_name, t.name, t.public, t.featured, " + "t.type, t.hvm, t.bits, t.url, t.format, t.created, t.account_id, t.checksum, t.display_text, " @@ -87,6 +93,7 @@ public class VMTemplateDaoImpl extends GenericDaoBase implem protected SearchBuilder UniqueNameSearch; protected SearchBuilder tmpltTypeSearch; protected SearchBuilder tmpltTypeHyperSearch; + protected SearchBuilder readySystemTemplateSearch; protected SearchBuilder tmpltTypeHyperSearch2; protected SearchBuilder AccountIdSearch; @@ -326,6 +333,23 @@ public class VMTemplateDaoImpl extends GenericDaoBase implem hostHyperSearch.done(); tmpltTypeHyperSearch.done(); + readySystemTemplateSearch = createSearchBuilder(); + readySystemTemplateSearch.and("templateType", readySystemTemplateSearch.entity().getTemplateType(), SearchCriteria.Op.EQ); + SearchBuilder templateDownloadSearch = _templateDataStoreDao.createSearchBuilder(); + templateDownloadSearch.and("downloadState", templateDownloadSearch.entity().getDownloadState(), SearchCriteria.Op.EQ); + readySystemTemplateSearch.join("vmTemplateJoinTemplateStoreRef", templateDownloadSearch, templateDownloadSearch.entity().getTemplateId(), + readySystemTemplateSearch.entity().getId(), JoinBuilder.JoinType.INNER); + SearchBuilder hostHyperSearch2 = _hostDao.createSearchBuilder(); + hostHyperSearch2.and("type", hostHyperSearch2.entity().getType(), SearchCriteria.Op.EQ); + hostHyperSearch2.and("zoneId", hostHyperSearch2.entity().getDataCenterId(), SearchCriteria.Op.EQ); + hostHyperSearch2.and("removed", hostHyperSearch2.entity().getRemoved(), SearchCriteria.Op.NULL); + hostHyperSearch2.groupBy(hostHyperSearch2.entity().getHypervisorType()); + + readySystemTemplateSearch.join("tmplHyper", hostHyperSearch2, hostHyperSearch2.entity().getHypervisorType(), + readySystemTemplateSearch.entity().getHypervisorType(), JoinBuilder.JoinType.INNER); + hostHyperSearch2.done(); + readySystemTemplateSearch.done(); + tmpltTypeHyperSearch2 = createSearchBuilder(); tmpltTypeHyperSearch2.and("templateType", tmpltTypeHyperSearch2.entity().getTemplateType(), SearchCriteria.Op.EQ); @@ -764,22 +788,27 @@ public class VMTemplateDaoImpl extends GenericDaoBase implem } @Override - public VMTemplateVO findSystemVMTemplate(long zoneId, HypervisorType hType) { - SearchCriteria sc = tmpltTypeHyperSearch.create(); + public VMTemplateVO findSystemVMReadyTemplate(long zoneId, HypervisorType hypervisorType) { + SearchCriteria sc = readySystemTemplateSearch.create(); sc.setParameters("templateType", Storage.TemplateType.SYSTEM); sc.setJoinParameters("tmplHyper", "type", Host.Type.Routing); sc.setJoinParameters("tmplHyper", "zoneId", zoneId); + sc.setJoinParameters("vmTemplateJoinTemplateStoreRef", "downloadState", VMTemplateStorageResourceAssoc.Status.DOWNLOADED); // order by descending order of id List tmplts = listBy(sc, new Filter(VMTemplateVO.class, "id", false, null, null)); - for (VMTemplateVO tmplt : tmplts) { - if (tmplt.getHypervisorType() == hType) { - return tmplt; + if (tmplts.size() > 0) { + if (hypervisorType == HypervisorType.Any) { + Collections.shuffle(tmplts); + return tmplts.get(0); } - } - if (tmplts.size() > 0 && hType == HypervisorType.Any) { - return tmplts.get(0); + for (VMTemplateVO tmplt : tmplts) { + if (tmplt.getHypervisorType() == hypervisorType) { + return tmplt; + } + } + } return null; } diff --git a/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java b/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java index b956f3ecb94..a58395457b0 100755 --- a/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java +++ b/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java @@ -645,9 +645,22 @@ public class ConsoleProxyManagerImpl extends ManagerBase implements ConsoleProxy s_logger.warn("The number of launched console proxy on zone " + dataCenterId + " has reached to limit"); return null; } - HypervisorType defaultHype = _resourceMgr.getAvailableHypervisor(dataCenterId); - Map context = createProxyInstance(dataCenterId, defaultHype); + VMTemplateVO template = null; + HypervisorType defaultHypervisor = _resourceMgr.getDefaultHypervisor(dataCenterId); + if (defaultHypervisor != HypervisorType.None) { + template = _templateDao.findSystemVMReadyTemplate(dataCenterId, defaultHypervisor); + if (template == null) { + throw new CloudRuntimeException("Not able to find the System template or not downloaded in zone " + dataCenterId + " corresponding to default System vm hypervisor " + defaultHypervisor); + } + } else { + template = _templateDao.findSystemVMReadyTemplate(dataCenterId, HypervisorType.Any); + if (template == null) { + throw new CloudRuntimeException("Not able to find the System templates or not downloaded in zone " + dataCenterId); + } + } + + Map context = createProxyInstance(dataCenterId, template); long proxyVmId = (Long) context.get("proxyVmId"); if (proxyVmId == 0) { @@ -673,7 +686,7 @@ public class ConsoleProxyManagerImpl extends ManagerBase implements ConsoleProxy return null; } - protected Map createProxyInstance(long dataCenterId, HypervisorType desiredHyp) throws ConcurrentOperationException { + protected Map createProxyInstance(long dataCenterId, VMTemplateVO template) throws ConcurrentOperationException { long id = _consoleProxyDao.getNextInSequence(Long.class, "id"); String name = VirtualMachineName.getConsoleProxyName(id, _instance); @@ -716,12 +729,6 @@ public class ConsoleProxyManagerImpl extends ManagerBase implements ConsoleProxy networks.add(new Pair(_networkMgr.setupNetwork(systemAcct, offering, plan, null, null, false).get(0), null)); } - VMTemplateVO template = _templateDao.findSystemVMTemplate(dataCenterId, desiredHyp); - if (template == null) { - s_logger.debug("Can't find a template to start"); - throw new CloudRuntimeException("Insufficient capacity exception"); - } - ConsoleProxyVO proxy = new ConsoleProxyVO(id, _serviceOffering.getId(), name, template.getId(), template.getHypervisorType(), template.getGuestOSId(), dataCenterId, systemAcct.getDomainId(), systemAcct.getId(), 0, _serviceOffering.getOfferHA()); proxy.setDynamicallyScalable(template.isDynamicallyScalable()); diff --git a/server/src/com/cloud/storage/secondary/SecondaryStorageManagerImpl.java b/server/src/com/cloud/storage/secondary/SecondaryStorageManagerImpl.java index 81aa07263a0..17cbf5c878c 100755 --- a/server/src/com/cloud/storage/secondary/SecondaryStorageManagerImpl.java +++ b/server/src/com/cloud/storage/secondary/SecondaryStorageManagerImpl.java @@ -34,6 +34,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope; import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; import org.apache.log4j.Logger; import com.cloud.agent.AgentManager; @@ -237,6 +238,8 @@ public class SecondaryStorageManagerImpl extends ManagerBase implements Secondar DataStoreManager _dataStoreMgr; @Inject ImageStoreDao _imageStoreDao; + @Inject + TemplateDataStoreDao _tmplStoreDao; private long _capacityScanInterval = DEFAULT_CAPACITY_SCAN_INTERVAL; private int _secStorageVmMtuSize; @@ -568,12 +571,18 @@ public class SecondaryStorageManagerImpl extends ManagerBase implements Secondar return new HashMap(); } - HypervisorType hypeType = _resourceMgr.getAvailableHypervisor(dataCenterId); - - VMTemplateVO template = _templateDao.findSystemVMTemplate(dataCenterId, hypeType); - if (template == null) { - s_logger.debug("Can't find a template to start"); - throw new CloudRuntimeException("Insufficient capacity exception"); + VMTemplateVO template = null; + HypervisorType defaultHypervisor = _resourceMgr.getDefaultHypervisor(dataCenterId); + if (defaultHypervisor != HypervisorType.None) { + template = _templateDao.findSystemVMReadyTemplate(dataCenterId, defaultHypervisor); + if (template == null) { + throw new CloudRuntimeException("Not able to find the System template or not downloaded in zone " + dataCenterId + " corresponding to default System vm hypervisor " + defaultHypervisor); + } + } else { + template = _templateDao.findSystemVMReadyTemplate(dataCenterId, HypervisorType.Any); + if (template == null) { + throw new CloudRuntimeException("Not able to find the System templates or not downloaded in zone " + dataCenterId); + } } SecondaryStorageVmVO secStorageVm = new SecondaryStorageVmVO(id, _serviceOffering.getId(), name, template.getId(), template.getHypervisorType(), template.getGuestOSId(), dataCenterId, From cd278cff440899bc472a399286c1852e1bd71fc4 Mon Sep 17 00:00:00 2001 From: Devdeep Singh Date: Mon, 5 Aug 2013 18:39:43 +0530 Subject: [PATCH 117/763] CLOUDSTACK-4079: Made a change to make sure we are setting the hypervisor capabilities for storage motion to true. If an upgrade is taking place from an older setup which supported a hypervisor but not storage motion on that hypervisor, the upgraded setup will not have the storage motion field enabled. Enabled it explicitly. --- setup/db/db/schema-410to420.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup/db/db/schema-410to420.sql b/setup/db/db/schema-410to420.sql index f3ce3368751..183b7391afd 100644 --- a/setup/db/db/schema-410to420.sql +++ b/setup/db/db/schema-410to420.sql @@ -37,6 +37,8 @@ INSERT IGNORE INTO `cloud`.`hypervisor_capabilities`(uuid, hypervisor_type, hype INSERT IGNORE INTO `cloud`.`hypervisor_capabilities`(uuid, hypervisor_type, hypervisor_version, max_guests_limit, security_group_enabled, max_hosts_per_cluster) VALUES (UUID(), 'VMware', '5.1', 128, 0, 32); UPDATE `cloud`.`hypervisor_capabilities` SET `storage_motion_supported`=true WHERE `hypervisor_type`='VMware' AND `hypervisor_version`='5.1'; UPDATE `cloud`.`hypervisor_capabilities` SET `storage_motion_supported`=true WHERE `hypervisor_type`='VMware' AND `hypervisor_version`='5.0'; +UPDATE `cloud`.`hypervisor_capabilities` SET `storage_motion_supported`=true WHERE `hypervisor_type`='XenServer' AND `hypervisor_version`='6.1.0'; +UPDATE `cloud`.`hypervisor_capabilities` SET `storage_motion_supported`=true WHERE `hypervisor_type`='XenServer' AND `hypervisor_version`='6.2.0'; DELETE FROM `cloud`.`configuration` where name='vmware.percluster.host.max'; DELETE FROM `cloud`.`configuration` where name='router.template.id'; DELETE FROM `cloud`.`configuration` where name='swift.enable'; From cb6ff00ef6f4e625a93d8e7ec4a1c2cf5895232a Mon Sep 17 00:00:00 2001 From: Murali Reddy Date: Mon, 5 Aug 2013 19:31:29 +0530 Subject: [PATCH 118/763] CLOUDSTACK-4048:[GSLB] Failed to assign more than one LB rule to a GSLB rule that is created with gslbmethod=leastconn Netscaler nitro api to add gslb virtual servers fails for some reason if both netmask and round robin methods are specified. So working around with settign netmask to be null while updating vserver. --- .../src/com/cloud/network/resource/NetscalerResource.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/plugins/network-elements/netscaler/src/com/cloud/network/resource/NetscalerResource.java b/plugins/network-elements/netscaler/src/com/cloud/network/resource/NetscalerResource.java index c82fde2cde9..e42a9eac2b5 100644 --- a/plugins/network-elements/netscaler/src/com/cloud/network/resource/NetscalerResource.java +++ b/plugins/network-elements/netscaler/src/com/cloud/network/resource/NetscalerResource.java @@ -1134,10 +1134,9 @@ public class NetscalerResource implements ServerResource { vserver.set_cookietimeout(null); vserver.set_domainname(null); if (isUpdateSite) { - if ("roundrobin".equalsIgnoreCase(lbMethod)) { - vserver.set_netmask(null); - vserver.set_v6netmasklen(null); - } + // both netmask and LB method can not be specified while update so set to null + vserver.set_netmask(null); + vserver.set_v6netmasklen(null); gslbvserver.update(client, vserver); } else { gslbvserver.add(client, vserver); From e0c6073a80ef5c1c8cff19b86289dc872b1b36f7 Mon Sep 17 00:00:00 2001 From: Sanjay Tripathi Date: Mon, 5 Aug 2013 18:13:52 +0530 Subject: [PATCH 119/763] CLOUDSTACK-4075: User unable to archive events --- server/src/com/cloud/server/ManagementServerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/com/cloud/server/ManagementServerImpl.java b/server/src/com/cloud/server/ManagementServerImpl.java index a1815537480..e534be02811 100755 --- a/server/src/com/cloud/server/ManagementServerImpl.java +++ b/server/src/com/cloud/server/ManagementServerImpl.java @@ -838,7 +838,7 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe boolean result =true; List permittedAccountIds = new ArrayList(); - if (caller.getType() == Account.ACCOUNT_TYPE_NORMAL && caller.getType() == Account.ACCOUNT_TYPE_PROJECT) { + if (caller.getType() == Account.ACCOUNT_TYPE_NORMAL || caller.getType() == Account.ACCOUNT_TYPE_PROJECT) { permittedAccountIds.add(caller.getId()); } else { DomainVO domain = _domainDao.findById(caller.getDomainId()); From f71eff7bfe0d3203712f22f57bae8e92c2fd8da2 Mon Sep 17 00:00:00 2001 From: Prasanna Santhanam Date: Mon, 5 Aug 2013 20:13:39 +0530 Subject: [PATCH 120/763] No need to cleanup lightweight entities like offerings Signed-off-by: Prasanna Santhanam --- .../component/test_redundant_router_network_rules.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/test/integration/component/test_redundant_router_network_rules.py b/test/integration/component/test_redundant_router_network_rules.py index f71954b599b..932e17af058 100644 --- a/test/integration/component/test_redundant_router_network_rules.py +++ b/test/integration/component/test_redundant_router_network_rules.py @@ -163,11 +163,6 @@ class TestRedundantRouterRulesLifeCycle(cloudstackTestCase): ) # Enable Network offering cls.network_offering.update(cls.api_client, state='Enabled') - - cls._cleanup = [ - cls.service_offering, - cls.network_offering, - ] return @classmethod From 5eb684214993b2912437a39b7caa5ec8792cf2ca Mon Sep 17 00:00:00 2001 From: Prasanna Santhanam Date: Mon, 5 Aug 2013 20:16:15 +0530 Subject: [PATCH 121/763] Log the API exception for debugging Signed-off-by: Prasanna Santhanam (cherry picked from commit f17e046319508ef9ccfe54095120cc2d4fe32c4a) --- .../component/test_redundant_router_network_rules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/component/test_redundant_router_network_rules.py b/test/integration/component/test_redundant_router_network_rules.py index 932e17af058..faed3898dff 100644 --- a/test/integration/component/test_redundant_router_network_rules.py +++ b/test/integration/component/test_redundant_router_network_rules.py @@ -1244,7 +1244,7 @@ class TestRedundantRouterRulesLifeCycle(cloudstackTestCase): try: Router.stop(self.apiclient, id=master_router.id) except Exception as e: - self.fail("Failed to stop master router..") + self.fail("Failed to stop master router becaues of %s" % e) self.debug("Associating public IP for network: %s" % network.name) public_ip = PublicIPAddress.create( From 78b745f7cda687925268e1694a2cae9943ec6638 Mon Sep 17 00:00:00 2001 From: Wido den Hollander Date: Mon, 5 Aug 2013 14:28:16 +0200 Subject: [PATCH 122/763] rbd: Move some logic from LibvirtComputingResource into KVMStorageProcessor All this logic was in LibvirtComputingResource but seems redundant. Move it into the KVMStorageProcessor --- .../kvm/storage/KVMStorageProcessor.java | 167 +++++++++++++++--- .../kvm/storage/LibvirtStoragePool.java | 2 +- 2 files changed, 139 insertions(+), 30 deletions(-) diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java index 783c65e078d..59563657d4a 100644 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java @@ -20,6 +20,9 @@ package com.cloud.hypervisor.kvm.storage; import java.io.File; import java.io.FileOutputStream; +import java.io.FileNotFoundException; +import java.io.BufferedOutputStream; +import java.io.IOException; import java.net.URISyntaxException; import java.text.DateFormat; import java.text.MessageFormat; @@ -50,6 +53,7 @@ import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat; import org.apache.cloudstack.utils.qemu.QemuImgException; import org.apache.cloudstack.utils.qemu.QemuImgFile; import org.apache.log4j.Logger; +import org.apache.commons.io.FileUtils; import org.libvirt.Connect; import org.libvirt.Domain; import org.libvirt.DomainInfo; @@ -81,6 +85,13 @@ import com.cloud.utils.NumbersUtil; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.script.Script; +import com.ceph.rados.Rados; +import com.ceph.rados.RadosException; +import com.ceph.rados.IoCTX; +import com.ceph.rbd.Rbd; +import com.ceph.rbd.RbdImage; +import com.ceph.rbd.RbdException; + public class KVMStorageProcessor implements StorageProcessor { private static final Logger s_logger = Logger.getLogger(KVMStorageProcessor.class); private KVMStoragePoolManager storagePoolMgr; @@ -445,16 +456,83 @@ public class KVMStorageProcessor implements StorageProcessor { KVMStoragePool primaryPool = storagePoolMgr.getStoragePool(primaryStore.getPoolType(), primaryStore.getUuid()); KVMPhysicalDisk snapshotDisk = primaryPool.getPhysicalDisk(volumePath); - Script command = new Script(_manageSnapshotPath, cmd.getWait() * 1000, s_logger); - command.add("-b", snapshotDisk.getPath()); - command.add("-n", snapshotName); - command.add("-p", snapshotDestPath); - command.add("-t", snapshotName); - String result = command.execute(); - if (result != null) { - s_logger.debug("Failed to backup snaptshot: " + result); - return new CopyCmdAnswer(result); + + /** + * RBD snapshots can't be copied using qemu-img, so we have to use + * the Java bindings for librbd here. + * + * These bindings will read the snapshot and write the contents to + * the secondary storage directly + * + * It will stop doing so if the amount of time spend is longer then + * cmds.timeout + */ + if (primaryPool.getType() == StoragePoolType.RBD) { + try { + Rados r = new Rados(primaryPool.getAuthUserName()); + r.confSet("mon_host", primaryPool.getSourceHost() + ":" + primaryPool.getSourcePort()); + r.confSet("key", primaryPool.getAuthSecret()); + r.connect(); + s_logger.debug("Succesfully connected to Ceph cluster at " + r.confGet("mon_host")); + + IoCTX io = r.ioCtxCreate(primaryPool.getSourceDir()); + Rbd rbd = new Rbd(io); + RbdImage image = rbd.open(snapshotDisk.getName(), snapshotName); + + long startTime = System.currentTimeMillis() / 1000; + + File snapDir = new File(snapshotDestPath); + s_logger.debug("Attempting to create " + snapDir.getAbsolutePath() + " recursively"); + FileUtils.forceMkdir(snapDir); + + File snapFile = new File(snapshotDestPath + "/" + snapshotName); + s_logger.debug("Backing up RBD snapshot to " + snapFile.getAbsolutePath()); + BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(snapFile)); + int chunkSize = 4194304; + long offset = 0; + while(true) { + byte[] buf = new byte[chunkSize]; + + int bytes = image.read(offset, buf, chunkSize); + if (bytes <= 0) { + break; + } + bos.write(buf, 0, bytes); + offset += bytes; + } + s_logger.debug("Completed backing up RBD snapshot " + snapshotName + " to " + snapFile.getAbsolutePath() + ". Bytes written: " + offset); + bos.close(); + + s_logger.debug("Attempting to remove snapshot RBD " + snapshotName + " from image " + snapshotDisk.getName()); + image.snapRemove(snapshotName); + + r.ioCtxDestroy(io); + } catch (RadosException e) { + s_logger.error("A RADOS operation failed. The error was: " + e.getMessage()); + return new CopyCmdAnswer(e.toString()); + } catch (RbdException e) { + s_logger.error("A RBD operation on " + snapshotDisk.getName() + " failed. The error was: " + e.getMessage()); + return new CopyCmdAnswer(e.toString()); + } catch (FileNotFoundException e) { + s_logger.error("Failed to open " + snapshotDestPath + ". The error was: " + e.getMessage()); + return new CopyCmdAnswer(e.toString()); + } catch (IOException e) { + s_logger.debug("An I/O error occured during a snapshot operation on " + snapshotDestPath); + return new CopyCmdAnswer(e.toString()); + } + } else { + Script command = new Script(_manageSnapshotPath, cmd.getWait() * 1000, s_logger); + command.add("-b", snapshotDisk.getPath()); + command.add("-n", snapshotName); + command.add("-p", snapshotDestPath); + command.add("-t", snapshotName); + String result = command.execute(); + if (result != null) { + s_logger.debug("Failed to backup snaptshot: " + result); + return new CopyCmdAnswer(result); + } } + /* Delete the snapshot on primary */ DomainInfo.DomainState state = null; @@ -484,13 +562,15 @@ public class KVMStorageProcessor implements StorageProcessor { vm.resume(); } } else { - command = new Script(_manageSnapshotPath, _cmdsTimeout, s_logger); - command.add("-d", snapshotDisk.getPath()); - command.add("-n", snapshotName); - result = command.execute(); - if (result != null) { - s_logger.debug("Failed to backup snapshot: " + result); - return new CopyCmdAnswer("Failed to backup snapshot: " + result); + if (primaryPool.getType() != StoragePoolType.RBD) { + Script command = new Script(_manageSnapshotPath, _cmdsTimeout, s_logger); + command.add("-d", snapshotDisk.getPath()); + command.add("-n", snapshotName); + String result = command.execute(); + if (result != null) { + s_logger.debug("Failed to backup snapshot: " + result); + return new CopyCmdAnswer("Failed to backup snapshot: " + result); + } } } @@ -767,11 +847,6 @@ public class KVMStorageProcessor implements StorageProcessor { KVMStoragePool primaryPool = storagePoolMgr.getStoragePool(primaryStore.getPoolType(), primaryStore.getUuid()); - if (primaryPool.getType() == StoragePoolType.RBD) { - s_logger.debug("Snapshots are not supported on RBD volumes"); - return new CreateObjectAnswer("Snapshots are not supported on RBD volumes"); - } - KVMPhysicalDisk disk = primaryPool.getPhysicalDisk(volume.getPath()); if (state == DomainInfo.DomainState.VIR_DOMAIN_RUNNING && !primaryPool.isExternalSnapshot()) { String vmUuid = vm.getUUIDString(); @@ -790,15 +865,49 @@ public class KVMStorageProcessor implements StorageProcessor { vm.resume(); } } else { + /** + * For RBD we can't use libvirt to do our snapshotting or any Bash scripts. + * libvirt also wants to store the memory contents of the Virtual Machine, + * but that's not possible with RBD since there is no way to store the memory + * contents in RBD. + * + * So we rely on the Java bindings for RBD to create our snapshot + * + * This snapshot might not be 100% consistent due to writes still being in the + * memory of the Virtual Machine, but if the VM runs a kernel which supports + * barriers properly (>2.6.32) this won't be any different then pulling the power + * cord out of a running machine. + */ + if (primaryPool.getType() == StoragePoolType.RBD) { + try { + Rados r = new Rados(primaryPool.getAuthUserName()); + r.confSet("mon_host", primaryPool.getSourceHost() + ":" + primaryPool.getSourcePort()); + r.confSet("key", primaryPool.getAuthSecret()); + r.connect(); + s_logger.debug("Succesfully connected to Ceph cluster at " + r.confGet("mon_host")); - /* VM is not running, create a snapshot by ourself */ - final Script command = new Script(_manageSnapshotPath, this._cmdsTimeout, s_logger); - command.add("-c", disk.getPath()); - command.add("-n", snapshotName); - String result = command.execute(); - if (result != null) { - s_logger.debug("Failed to manage snapshot: " + result); - return new CreateObjectAnswer("Failed to manage snapshot: " + result); + IoCTX io = r.ioCtxCreate(primaryPool.getSourceDir()); + Rbd rbd = new Rbd(io); + RbdImage image = rbd.open(disk.getName()); + + s_logger.debug("Attempting to create RBD snapshot " + disk.getName() + "@" + snapshotName); + image.snapCreate(snapshotName); + + rbd.close(image); + r.ioCtxDestroy(io); + } catch (Exception e) { + s_logger.error("A RBD snapshot operation on " + disk.getName() + " failed. The error was: " + e.getMessage()); + } + } else { + /* VM is not running, create a snapshot by ourself */ + final Script command = new Script(_manageSnapshotPath, this._cmdsTimeout, s_logger); + command.add("-c", disk.getPath()); + command.add("-n", snapshotName); + String result = command.execute(); + if (result != null) { + s_logger.debug("Failed to manage snapshot: " + result); + return new CreateObjectAnswer("Failed to manage snapshot: " + result); + } } } diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/LibvirtStoragePool.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/LibvirtStoragePool.java index bed7b1f652d..2ce517504d6 100644 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/LibvirtStoragePool.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/LibvirtStoragePool.java @@ -140,7 +140,7 @@ public class LibvirtStoragePool implements KVMStoragePool { @Override public boolean isExternalSnapshot() { - if (this.type == StoragePoolType.Filesystem || this.type == StoragePoolType.RBD) { + if (this.type == StoragePoolType.Filesystem) { return false; } From f025db9582b02de4bce042e9d73279d472033c90 Mon Sep 17 00:00:00 2001 From: Wido den Hollander Date: Mon, 5 Aug 2013 17:37:57 +0200 Subject: [PATCH 123/763] config: Only user low-case names for variables --- server/src/com/cloud/configuration/Config.java | 2 +- setup/db/db/schema-410to420.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/com/cloud/configuration/Config.java b/server/src/com/cloud/configuration/Config.java index 7bc7193aa03..c146dd28569 100755 --- a/server/src/com/cloud/configuration/Config.java +++ b/server/src/com/cloud/configuration/Config.java @@ -145,7 +145,7 @@ public enum Config { SnapshotPollInterval("Snapshots", SnapshotManager.class, Integer.class, "snapshot.poll.interval", "300", "The time interval in seconds when the management server polls for snapshots to be scheduled.", null), SnapshotDeltaMax("Snapshots", SnapshotManager.class, Integer.class, "snapshot.delta.max", "16", "max delta snapshots between two full snapshots.", null), BackupSnapshotAferTakingSnapshot("Snapshots", SnapshotManager.class, Boolean.class, "snapshot.backup.rightafter", "true", "backup snapshot right after snapshot is taken", null), - KVMSnapshotEnabled("Snapshots", SnapshotManager.class, Boolean.class, "KVM.snapshot.enabled", "false", "whether snapshot is enabled for KVM hosts", null), + KVMSnapshotEnabled("Snapshots", SnapshotManager.class, Boolean.class, "kvm.snapshot.enabled", "false", "whether snapshot is enabled for KVM hosts", null), // Advanced JobExpireMinutes("Advanced", ManagementServer.class, String.class, "job.expire.minutes", "1440", "Time (in minutes) for async-jobs to be kept in system", null), diff --git a/setup/db/db/schema-410to420.sql b/setup/db/db/schema-410to420.sql index 183b7391afd..f98bb500631 100644 --- a/setup/db/db/schema-410to420.sql +++ b/setup/db/db/schema-410to420.sql @@ -2236,6 +2236,6 @@ CREATE VIEW `cloud`.`account_vmstats_view` AS group by account_id , state; INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Advanced', 'DEFAULT', 'management-server', 'healthcheck.update.interval', '600', 'Time Interval to fetch the LB health check states (in sec)'); -INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Snapshots', 'DEFAULT', 'SnapshotManager', 'KVM.snapshot.enabled', 'false', 'whether snapshot is enabled for KVM hosts'); +INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Snapshots', 'DEFAULT', 'SnapshotManager', 'kvm.snapshot.enabled', 'false', 'whether snapshot is enabled for KVM hosts'); INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Advanced', 'DEFAULT', 'management-server', 'eip.use.multiple.netscalers', 'false', 'Should be set to true, if there will be multiple NetScaler devices providing EIP service in a zone'); INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Snapshots', 'DEFAULT', 'SnapshotManager', 'snapshot.backup.rightafter', 'true', 'backup snapshot right after snapshot is taken'); From 913a0050e4f73a698d7228cd38be5d30124c4379 Mon Sep 17 00:00:00 2001 From: Alena Prokharchyk Date: Mon, 5 Aug 2013 09:00:54 -0700 Subject: [PATCH 124/763] CLOUDSTACK-4065: db upgrade - when insert placeHolder nic for Shared networks, always mark this nic as non-default explicitly --- .../cloud/upgrade/dao/Upgrade410to420.java | 32 +++++++++---------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/engine/schema/src/com/cloud/upgrade/dao/Upgrade410to420.java b/engine/schema/src/com/cloud/upgrade/dao/Upgrade410to420.java index 2e5fbcbc7b7..5537a496737 100644 --- a/engine/schema/src/com/cloud/upgrade/dao/Upgrade410to420.java +++ b/engine/schema/src/com/cloud/upgrade/dao/Upgrade410to420.java @@ -17,16 +17,6 @@ package com.cloud.upgrade.dao; -import com.cloud.deploy.DeploymentPlanner; -import com.cloud.storage.Upload; -import com.cloud.storage.UploadVO; -import com.cloud.utils.DateUtil; -import com.cloud.utils.exception.CloudRuntimeException; -import com.cloud.utils.script.Script; -import com.cloud.hypervisor.Hypervisor.HypervisorType; - -import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProvider; -import org.apache.log4j.Logger; import java.io.File; import java.sql.Connection; import java.sql.Date; @@ -35,13 +25,21 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Types; import java.util.ArrayList; -import java.util.List; -import java.util.UUID; -import java.util.Set; -import java.util.HashSet; -import java.util.Map; import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; + +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProvider; +import org.apache.log4j.Logger; + +import com.cloud.deploy.DeploymentPlanner; +import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.network.vpc.NetworkACL; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.script.Script; public class Upgrade410to420 implements DbUpgrade { final static Logger s_logger = Logger.getLogger(Upgrade410to420.class); @@ -756,13 +754,13 @@ public class Upgrade410to420 implements DbUpgrade { String ip = rs.getString(3); String uuid = UUID.randomUUID().toString(); //Insert placeholder nic for each Domain router nic in Shared network - pstmt = conn.prepareStatement("INSERT INTO `cloud`.`nics` (uuid, ip4_address, gateway, network_id, state, strategy, vm_type) VALUES (?, ?, ?, ?, 'Reserved', 'PlaceHolder', 'DomainRouter')"); + pstmt = conn.prepareStatement("INSERT INTO `cloud`.`nics` (uuid, ip4_address, gateway, network_id, state, strategy, vm_type, default_nic, created) VALUES (?, ?, ?, ?, 'Reserved', 'PlaceHolder', 'DomainRouter', 0, now())"); pstmt.setString(1, uuid); pstmt.setString(2, ip); pstmt.setString(3, gateway); pstmt.setLong(4, networkId); pstmt.executeUpdate(); - s_logger.debug("Created placeholder nic for the ipAddress " + ip); + s_logger.debug("Created placeholder nic for the ipAddress " + ip + " and network " + networkId); } } catch (SQLException e) { From 04ba3d1354831ed30a0b43889b5e8849d352b4ec Mon Sep 17 00:00:00 2001 From: Prasanna Santhanam Date: Mon, 5 Aug 2013 22:04:30 +0530 Subject: [PATCH 125/763] remove hardcoded vlan for shared network test use a vlan outside of the vlans managed by cloudstack. Signed-off-by: Prasanna Santhanam (cherry picked from commit 7eefeab957e7b2a904a2f5ec1ce2a2624587eb9e) --- test/integration/component/test_shared_networks.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/test/integration/component/test_shared_networks.py b/test/integration/component/test_shared_networks.py index 237e0bfc693..23c8d9e7b2e 100644 --- a/test/integration/component/test_shared_networks.py +++ b/test/integration/component/test_shared_networks.py @@ -1821,7 +1821,7 @@ class TestSharedNetworks(cloudstackTestCase): ) physical_network = list_physical_networks_response[0] - + self.debug("Physical Network found: %s" % physical_network.id) self.services["network_offering"]["specifyVlan"] = "True" @@ -1980,7 +1980,11 @@ class TestSharedNetworks(cloudstackTestCase): ) physical_network = list_physical_networks_response[0] - + start_vlan, end_vlan = physical_network.vlan.split('-') + self.assert_(int(start_vlan) < int(end_vlan), "VLAN range %s was improperly split" % physical_network.vlan) + shared_ntwk_vlan = int(end_vlan) + 1 + self.assert_(shared_ntwk_vlan < 4095, "VLAN chosen %s is invalid > 4095" % shared_ntwk_vlan) + self.debug("Physical Network found: %s" % physical_network.id) self.services["network_offering"]["specifyVlan"] = "True" @@ -2050,7 +2054,8 @@ class TestSharedNetworks(cloudstackTestCase): self.services["network"]["acltype"] = "Domain" self.services["network"]["networkofferingid"] = self.shared_network_offering.id self.services["network"]["physicalnetworkid"] = physical_network.id - self.services["network"]["vlan"] = "567" + self.services["network"]["vlan"] = shared_ntwk_vlan + self.debug("Creating a shared network in non-cloudstack VLAN %s" % shared_ntwk_vlan) self.network = Network.create( self.api_client, self.services["network"], From fda366f8d6e83e4a40da2630c87972d09cfa620e Mon Sep 17 00:00:00 2001 From: Alena Prokharchyk Date: Mon, 5 Aug 2013 10:08:29 -0700 Subject: [PATCH 126/763] CLOUDSTACK-4087: updateTemplatePermissions - derive domainId from the template owner, not from the operation caller --- .../cloud/template/TemplateManagerImpl.java | 67 +++++++++++-------- 1 file changed, 38 insertions(+), 29 deletions(-) diff --git a/server/src/com/cloud/template/TemplateManagerImpl.java b/server/src/com/cloud/template/TemplateManagerImpl.java index d2b2afef3df..a5f389de1a8 100755 --- a/server/src/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/com/cloud/template/TemplateManagerImpl.java @@ -25,11 +25,11 @@ import java.util.UUID; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; + import javax.ejb.Local; import javax.inject.Inject; import javax.naming.ConfigurationException; -import com.cloud.utils.DateUtil; import org.apache.cloudstack.acl.SecurityChecker.AccessType; import org.apache.cloudstack.api.BaseListTemplateOrIsoPermissionsCmd; import org.apache.cloudstack.api.BaseUpdateTemplateOrIsoCmd; @@ -48,8 +48,22 @@ import org.apache.cloudstack.api.command.user.template.ListTemplatePermissionsCm import org.apache.cloudstack.api.command.user.template.RegisterTemplateCmd; import org.apache.cloudstack.api.command.user.template.UpdateTemplateCmd; import org.apache.cloudstack.api.command.user.template.UpdateTemplatePermissionsCmd; -import org.apache.cloudstack.engine.subsystem.api.storage.*; +import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; +import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; +import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector; +import org.apache.cloudstack.engine.subsystem.api.storage.Scope; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; +import org.apache.cloudstack.engine.subsystem.api.storage.StorageCacheManager; +import org.apache.cloudstack.engine.subsystem.api.storage.TemplateDataFactory; +import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo; +import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService; import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService.TemplateApiResult; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; +import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope; import org.apache.cloudstack.framework.async.AsyncCallFuture; import org.apache.cloudstack.storage.command.AttachCommand; import org.apache.cloudstack.storage.command.CommandResult; @@ -67,7 +81,6 @@ import com.cloud.agent.AgentManager; import com.cloud.agent.api.Answer; import com.cloud.agent.api.Command; import com.cloud.agent.api.ComputeChecksumCommand; - import com.cloud.agent.api.storage.DestroyCommand; import com.cloud.agent.api.to.DataTO; import com.cloud.agent.api.to.DiskTO; @@ -83,6 +96,7 @@ import com.cloud.dc.DataCenter; import com.cloud.dc.DataCenterVO; import com.cloud.dc.dao.ClusterDao; import com.cloud.dc.dao.DataCenterDao; +import com.cloud.domain.Domain; import com.cloud.domain.dao.DomainDao; import com.cloud.event.ActionEvent; import com.cloud.event.EventTypes; @@ -101,32 +115,29 @@ import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.hypervisor.HypervisorGuruManager; import com.cloud.projects.Project; import com.cloud.projects.ProjectManager; - import com.cloud.resource.ResourceManager; import com.cloud.server.ConfigurationServer; +import com.cloud.storage.DataStoreRole; import com.cloud.storage.GuestOSVO; import com.cloud.storage.LaunchPermissionVO; +import com.cloud.storage.ScopeType; import com.cloud.storage.Snapshot; import com.cloud.storage.SnapshotVO; import com.cloud.storage.Storage; - import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.Storage.TemplateType; -import com.cloud.storage.DataStoreRole; -import com.cloud.storage.ScopeType; import com.cloud.storage.StorageManager; import com.cloud.storage.StoragePool; import com.cloud.storage.StoragePoolHostVO; import com.cloud.storage.StoragePoolStatus; import com.cloud.storage.TemplateProfile; import com.cloud.storage.Upload; -import com.cloud.storage.VMTemplateZoneVO; - import com.cloud.storage.VMTemplateHostVO; import com.cloud.storage.VMTemplateStoragePoolVO; import com.cloud.storage.VMTemplateStorageResourceAssoc; import com.cloud.storage.VMTemplateStorageResourceAssoc.Status; import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.VMTemplateZoneVO; import com.cloud.storage.Volume; import com.cloud.storage.VolumeManager; import com.cloud.storage.VolumeVO; @@ -145,7 +156,6 @@ import com.cloud.storage.download.DownloadMonitor; import com.cloud.storage.secondary.SecondaryStorageVmManager; import com.cloud.storage.upload.UploadMonitor; import com.cloud.template.TemplateAdapter.TemplateAdapterType; - import com.cloud.user.Account; import com.cloud.user.AccountManager; import com.cloud.user.AccountService; @@ -157,13 +167,15 @@ import com.cloud.user.dao.AccountDao; import com.cloud.user.dao.UserAccountDao; import com.cloud.user.dao.UserDao; import com.cloud.uservm.UserVm; +import com.cloud.utils.DateUtil; import com.cloud.utils.EnumUtils; import com.cloud.utils.NumbersUtil; import com.cloud.utils.Pair; import com.cloud.utils.component.AdapterBase; import com.cloud.utils.component.ManagerBase; import com.cloud.utils.concurrency.NamedThreadFactory; -import com.cloud.utils.db.*; +import com.cloud.utils.db.DB; +import com.cloud.utils.db.Transaction; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.UserVmManager; import com.cloud.vm.UserVmVO; @@ -385,9 +397,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, if (template == null) { throw new InvalidParameterValueException("unable to find template with id " + templateId); } - TemplateAdapter adapter = getAdapter(template.getHypervisorType()); - TemplateProfile profile = adapter.prepareExtractTemplate(cmd); - + return extract(caller, templateId, url, zoneId, mode, eventId, false); } @@ -711,7 +721,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, throw new InvalidParameterValueException("There is no template " + templateId + " in zone " + sourceZoneId); } if (srcSecStore.getScope().getScopeType() == ScopeType.REGION) { - s_logger.debug("Template " + templateId + " is in region-wide secondary storage " + dstSecStore.getName() + " , don't need to copy"); + s_logger.debug("Template " + templateId + " is in region-wide secondary storage " + srcSecStore.getName() + " , don't need to copy"); return template; } @@ -809,9 +819,6 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, @Override public boolean configure(String name, Map params) throws ConfigurationException { - - final Map configs = _configDao.getConfiguration("AgentManager", params); - String value = _configDao.getValue(Config.PrimaryStorageDownloadWait.toString()); _primaryStorageDownloadWait = NumbersUtil.parseInt(value, Integer.parseInt(Config.PrimaryStorageDownloadWait.getDefaultValue())); @@ -1237,8 +1244,8 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, } } - Long accountId = template.getAccountId(); - if (accountId == null) { + Long ownerId = template.getAccountId(); + if (ownerId == null) { // if there is no owner of the template then it's probably already a // public template (or domain private template) so // publishing to individual users is irrelevant @@ -1270,12 +1277,19 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, } _tmpltDao.update(template.getId(), updatedTemplate); + + //when operation is add/remove, accountNames can not be null + if (("add".equalsIgnoreCase(operation) || "remove".equalsIgnoreCase(operation)) && accountNames == null) { + throw new InvalidParameterValueException("Operation " + operation + " requires accounts or projectIds to be passed in"); + } - Long domainId = caller.getDomainId(); + //Derive the domain id from the template owner as updateTemplatePermissions is not cross domain operation + Account owner = _accountMgr.getAccount(ownerId); + Domain domain = _domainDao.findById(owner.getDomainId()); if ("add".equalsIgnoreCase(operation)) { txn.start(); for (String accountName : accountNames) { - Account permittedAccount = _accountDao.findActiveAccount(accountName, domainId); + Account permittedAccount = _accountDao.findActiveAccount(accountName, domain.getId()); if (permittedAccount != null) { if (permittedAccount.getId() == caller.getId()) { continue; // don't grant permission to the template @@ -1288,7 +1302,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, } } else { txn.rollback(); - throw new InvalidParameterValueException("Unable to grant a launch permission to account " + accountName + throw new InvalidParameterValueException("Unable to grant a launch permission to account " + accountName + " in domain id=" + domain.getUuid() + ", account not found. " + "No permissions updated, please verify the account names and retry."); } } @@ -1296,7 +1310,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, } else if ("remove".equalsIgnoreCase(operation)) { List accountIds = new ArrayList(); for (String accountName : accountNames) { - Account permittedAccount = _accountDao.findActiveAccount(accountName, domainId); + Account permittedAccount = _accountDao.findActiveAccount(accountName, domain.getId()); if (permittedAccount != null) { accountIds.add(permittedAccount.getId()); } @@ -1314,10 +1328,6 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, return true; } - private String getRandomPrivateTemplateName() { - return UUID.randomUUID().toString(); - } - @Override @DB @ActionEvent(eventType = EventTypes.EVENT_TEMPLATE_CREATE, eventDescription = "creating template", async = true) @@ -1526,7 +1536,6 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, } volume = this._volumeDao.findById(snapshot.getVolumeId()); - VolumeVO snapshotVolume = this._volumeDao.findByIdIncludingRemoved(snapshot.getVolumeId()); // check permissions _accountMgr.checkAccess(caller, null, true, snapshot); From 001ff45a76a071ae1a2b0a543e2c9716a05120d2 Mon Sep 17 00:00:00 2001 From: frank Date: Mon, 5 Aug 2013 11:09:51 -0700 Subject: [PATCH 127/763] CloudStack CLOUDSTACK-3307 UCS:DB: Same UCS manager can be added multiple times --- .../com/cloud/ucs/manager/UcsManagerImpl.java | 59 ++++++++++--------- 1 file changed, 32 insertions(+), 27 deletions(-) diff --git a/plugins/hypervisors/ucs/src/com/cloud/ucs/manager/UcsManagerImpl.java b/plugins/hypervisors/ucs/src/com/cloud/ucs/manager/UcsManagerImpl.java index fdc09d357fc..e19c72aef00 100755 --- a/plugins/hypervisors/ucs/src/com/cloud/ucs/manager/UcsManagerImpl.java +++ b/plugins/hypervisors/ucs/src/com/cloud/ucs/manager/UcsManagerImpl.java @@ -208,36 +208,41 @@ public class UcsManagerImpl implements UcsManager { @Override @DB - public UcsManagerResponse addUcsManager(AddUcsManagerCmd cmd) { - SearchCriteriaService q = SearchCriteria2.create(UcsManagerVO.class); - q.addAnd(q.getEntity().getUrl(), Op.EQ, cmd.getUrl()); - UcsManagerVO mgrvo = q.find(); - if (mgrvo != null) { - throw new IllegalArgumentException(String.format("duplicate UCS manager. url[%s] is used by another UCS manager already", cmd.getUrl())); - } - - UcsManagerVO vo = new UcsManagerVO(); - vo.setUuid(UUID.randomUUID().toString()); - vo.setPassword(cmd.getPassword()); - vo.setUrl(cmd.getUrl()); - vo.setUsername(cmd.getUsername()); - vo.setZoneId(cmd.getZoneId()); - vo.setName(cmd.getName()); + public UcsManagerResponse addUcsManager(AddUcsManagerCmd cmd) { + SearchCriteriaService q = SearchCriteria2.create(UcsManagerVO.class); + q.addAnd(q.getEntity().getUrl(), Op.EQ, cmd.getUrl()); + UcsManagerVO mgrvo = q.find(); + if (mgrvo != null) { + throw new IllegalArgumentException(String.format("duplicate UCS manager. url[%s] is used by another UCS manager already", cmd.getUrl())); + } + try { + UcsManagerVO vo = new UcsManagerVO(); + vo.setUuid(UUID.randomUUID().toString()); + vo.setPassword(cmd.getPassword()); + vo.setUrl(cmd.getUrl()); + vo.setUsername(cmd.getUsername()); + vo.setZoneId(cmd.getZoneId()); + vo.setName(cmd.getName()); - Transaction txn = Transaction.currentTxn(); - txn.start(); - ucsDao.persist(vo); - txn.commit(); - UcsManagerResponse rsp = new UcsManagerResponse(); - rsp.setId(String.valueOf(vo.getId())); - rsp.setName(vo.getName()); - rsp.setUrl(vo.getUrl()); - rsp.setZoneId(String.valueOf(vo.getZoneId())); + Transaction txn = Transaction.currentTxn(); + txn.start(); + mgrvo = ucsDao.persist(vo); + txn.commit(); + UcsManagerResponse rsp = new UcsManagerResponse(); + rsp.setId(String.valueOf(vo.getId())); + rsp.setName(vo.getName()); + rsp.setUrl(vo.getUrl()); + rsp.setZoneId(String.valueOf(vo.getZoneId())); - discoverBlades(vo); - - return rsp; + discoverBlades(vo); + return rsp; + } catch (CloudRuntimeException e) { + if (mgrvo != null) { + ucsDao.remove(mgrvo.getId()); + } + throw e; + } } private String getCookie(Long ucsMgrId) { From 0f539b4ce1b07c672a776ffa806dabb70c528afe Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Mon, 5 Aug 2013 11:21:00 -0700 Subject: [PATCH 128/763] CLOUDSTACK-2729: use file lock to prevent concurrent refreshPool/deleteVolume on KVM shared storage pool Signed-off-by: Edison Su --- .../kvm/storage/LibvirtStorageAdaptor.java | 72 +++++++++++++++++-- 1 file changed, 67 insertions(+), 5 deletions(-) diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java index e77916255e9..22a1be9e8fd 100644 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java @@ -72,6 +72,7 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { private StorageLayer _storageLayer; private String _mountPoint = "/mnt"; private String _manageSnapshotPath; + private String _lockfile = "KVMFILELOCK" + File.separator + ".lock"; private String rbdTemplateSnapName = "cloudstack-base-snap"; private int rbdFeatures = (1<<0); /* Feature 1<<0 means layering in RBD format 2 */ @@ -123,7 +124,7 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { public void storagePoolRefresh(StoragePool pool) { try { synchronized (getStoragePool(pool.getUUIDString())) { - pool.refresh(0); + refreshPool(pool); } } catch (LibvirtException e) { @@ -359,8 +360,9 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { } LibvirtStoragePoolDef spd = getStoragePoolDef(conn, storage); StoragePoolType type = null; - if (spd.getPoolType() == LibvirtStoragePoolDef.poolType.NETFS - || spd.getPoolType() == LibvirtStoragePoolDef.poolType.DIR) { + if (spd.getPoolType() == LibvirtStoragePoolDef.poolType.NETFS) { + type = StoragePoolType.NetworkFilesystem; + } else if (spd.getPoolType() == LibvirtStoragePoolDef.poolType.DIR) { type = StoragePoolType.Filesystem; } else if (spd.getPoolType() == LibvirtStoragePoolDef.poolType.RBD) { type = StoragePoolType.RBD; @@ -731,7 +733,7 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { LibvirtStoragePool libvirtPool = (LibvirtStoragePool) pool; try { StorageVol vol = this.getVolume(libvirtPool.getPool(), uuid); - vol.delete(0); + deleteVol(libvirtPool, vol); vol.free(); return true; } catch (LibvirtException e) { @@ -1131,7 +1133,7 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { LibvirtStoragePool libvirtPool = (LibvirtStoragePool) pool; StoragePool virtPool = libvirtPool.getPool(); try { - virtPool.refresh(0); + refreshPool(virtPool); } catch (LibvirtException e) { return false; } @@ -1159,4 +1161,64 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { return true; } + // refreshPool and deleteVol are used to fix CLOUDSTACK-2729/CLOUDSTACK-2780 + // They are caused by a libvirt bug (https://bugzilla.redhat.com/show_bug.cgi?id=977706) + // However, we also need to fix the issues in CloudStack source code. + // A file lock is used to prevent deleting a volume from a KVM storage pool when refresh it. + private void refreshPool(StoragePool pool) throws LibvirtException { + Connect conn = LibvirtConnection.getConnection(); + LibvirtStoragePoolDef spd = getStoragePoolDef(conn, pool); + if ((! spd.getPoolType().equals(LibvirtStoragePoolDef.poolType.NETFS)) + && (! spd.getPoolType().equals(LibvirtStoragePoolDef.poolType.DIR))) { + pool.refresh(0); + return; + } + String lockFile = _mountPoint + File.separator + pool.getUUIDString() + File.separator + _lockfile; + if (lock(lockFile, 30)) { + pool.refresh(0); + unlock(lockFile); + } else { + throw new CloudRuntimeException("Can not get file lock to refresh the pool" + pool.getUUIDString()); + } + } + + private void deleteVol(LibvirtStoragePool pool, StorageVol vol) throws LibvirtException { + if ((! pool.getType().equals(StoragePoolType.NetworkFilesystem)) + && (! pool.getType().equals(StoragePoolType.Filesystem))) { + vol.delete(0); + return; + } + String lockFile = pool.getLocalPath() + File.separator + _lockfile; + if (lock(lockFile, 30)) { + vol.delete(0); + unlock(lockFile); + } else { + throw new CloudRuntimeException("Can not get file lock to delete the volume" + vol.getPath()); + } + } + + private boolean lock(String path, int wait) { + File lockFile = new File(path); + lockFile.getParentFile().mkdir(); + boolean havelock = false; + try { + while (wait > 0) { + if (lockFile.createNewFile()) { + havelock = true; + break; + } + s_logger.debug("lockFile " + _lockfile + " already exists, waiting 1000 ms"); + Thread.sleep(1000); + wait--; + } + } catch (IOException e) { + } catch (InterruptedException e) { + } + return havelock; + } + + private void unlock(String path) { + File lockFile = new File(path); + lockFile.delete(); + } } From ff803f4131d6fe170004608da272b88f5a991202 Mon Sep 17 00:00:00 2001 From: Daan Hoogland Date: Sat, 3 Aug 2013 11:46:35 +0200 Subject: [PATCH 129/763] fix warnings in NetworkServiceImpl: removed unused code and add cases to switches Signed-off-by: Sheng Yang --- .../src/com/cloud/network/NetworkServiceImpl.java | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/server/src/com/cloud/network/NetworkServiceImpl.java b/server/src/com/cloud/network/NetworkServiceImpl.java index 739f5cc9978..304cefa90bc 100755 --- a/server/src/com/cloud/network/NetworkServiceImpl.java +++ b/server/src/com/cloud/network/NetworkServiceImpl.java @@ -547,7 +547,7 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService { long callerUserId = UserContext.current().getCallerUserId(); DataCenter zone = _configMgr.getZone(zoneId); - if ((networkId == null && vpcId == null) && (networkId != null && vpcId != null)) { + if ((networkId == null && vpcId == null) || (networkId != null && vpcId != null)) { throw new InvalidParameterValueException("One of Network id or VPC is should be passed"); } @@ -651,7 +651,6 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService { throw new InvalidParameterValueException("The nic is not belongs to user vm"); } - Nic nic = _nicDao.findById(nicId); VirtualMachine vm = _userVmDao.findById(nicVO.getInstanceId()); if (vm == null) { throw new InvalidParameterValueException("There is no vm with the nic"); @@ -671,7 +670,6 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService { NetworkOfferingVO ntwkOff = _networkOfferingDao.findById(network.getNetworkOfferingId()); DataCenter dc = _dcDao.findById(network.getDataCenterId()); - Long id = nicVO.getInstanceId(); DataCenter zone = _configMgr.getZone(zoneId); if (zone == null) { @@ -3337,12 +3335,8 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService { boolean update = false; if (state != null) { - if (state == PhysicalNetworkServiceProvider.State.Shutdown) { - throw new InvalidParameterValueException("Updating the provider state to 'Shutdown' is not supported"); - } - if (s_logger.isDebugEnabled()) { - s_logger.debug("updating state of the service provider id=" + id + " on physical network: " + provider.getPhysicalNetworkId() + " to state: " + stateStr); + s_logger.debug("trying to update the state of the service provider id=" + id + " on physical network: " + provider.getPhysicalNetworkId() + " to state: " + stateStr); } switch (state) { case Enabled: @@ -3358,6 +3352,8 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService { provider.setState(PhysicalNetworkServiceProvider.State.Disabled); update = true; break; + case Shutdown: + throw new InvalidParameterValueException("Updating the provider state to 'Shutdown' is not supported"); } } @@ -3565,6 +3561,9 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService { case Control: xenLabel = "cloud_link_local_network"; break; + case Vpn: + case None: + break; } return xenLabel; } From d1ede5430c052de5215dc1e7819d0d94752ede52 Mon Sep 17 00:00:00 2001 From: Brian Federle Date: Mon, 5 Aug 2013 11:42:25 -0700 Subject: [PATCH 130/763] CLOUDSTACK-2340: Display service state for health-checked VMs --- client/WEB-INF/classes/resources/messages.properties | 1 + ui/dictionary.jsp | 1 + ui/scripts/network.js | 5 +++++ ui/scripts/ui/widgets/multiEdit.js | 11 ++++++++--- 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/client/WEB-INF/classes/resources/messages.properties b/client/WEB-INF/classes/resources/messages.properties index 72c66c6b9e4..3920934a948 100644 --- a/client/WEB-INF/classes/resources/messages.properties +++ b/client/WEB-INF/classes/resources/messages.properties @@ -14,6 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +label.service.state=Service State label.egress.default.policy=Egress Default Policy label.routing=Routing label.about=About diff --git a/ui/dictionary.jsp b/ui/dictionary.jsp index 947ae7d8718..3f58c5937c3 100644 --- a/ui/dictionary.jsp +++ b/ui/dictionary.jsp @@ -25,6 +25,7 @@ under the License. <% long now = System.currentTimeMillis(); %>