From 17366f2b182aaa0943be451c79dac36fc08e221b Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Thu, 6 Aug 2015 12:13:01 +0530 Subject: [PATCH] CLOUDSTACK-8301: Enable configuring local storage use for system VMs at zone level Backported from #263 for 4.5 branch, original bugfix by @koushik-das et al More information on: https://issues.apache.org/jira/browse/CLOUDSTACK-8301 https://cwiki.apache.org/confluence/display/CLOUDSTACK/Enable+configuring+local+storage+use+for+system+VMs+at+zone+level Signed-off-by: Rohit Yadav This closes #661 --- api/src/com/cloud/dc/DataCenter.java | 4 - .../classes/resources/messages.properties | 3 +- .../cloud/service/dao/ServiceOfferingDao.java | 14 +- .../service/dao/ServiceOfferingDaoImpl.java | 53 ++ .../lb/ElasticLoadBalancerManagerImpl.java | 20 +- .../network/lb/LoadBalanceRuleHandler.java | 16 +- .../lb/InternalLoadBalancerVMManagerImpl.java | 24 +- .../InternalLBVMManagerTest.java | 7 +- .../InternalLBVMServiceTest.java | 9 +- .../src/com/cloud/configuration/Config.java | 1 - .../ConfigurationManagerImpl.java | 45 +- .../consoleproxy/ConsoleProxyManagerImpl.java | 39 +- .../deploy/DeploymentPlanningManagerImpl.java | 35 +- .../VirtualNetworkApplianceManagerImpl.java | 15 +- ...VpcVirtualNetworkApplianceManagerImpl.java | 7 +- .../com/cloud/storage/StorageManagerImpl.java | 14 +- .../element/VirtualRouterElementTest.java | 3 + .../SecondaryStorageManagerImpl.java | 35 +- .../test_zone_level_local_storage_setting.py | 734 ++++++++++++++++++ ui/dictionary2.jsp | 3 +- ui/scripts/zoneWizard.js | 94 ++- 21 files changed, 998 insertions(+), 177 deletions(-) create mode 100644 test/integration/component/maint/test_zone_level_local_storage_setting.py diff --git a/api/src/com/cloud/dc/DataCenter.java b/api/src/com/cloud/dc/DataCenter.java index 6cd054e28bc..5b3d3c01f30 100644 --- a/api/src/com/cloud/dc/DataCenter.java +++ b/api/src/com/cloud/dc/DataCenter.java @@ -20,7 +20,6 @@ import com.cloud.org.Grouping; import org.apache.cloudstack.acl.InfrastructureEntity; import org.apache.cloudstack.api.Identity; import org.apache.cloudstack.api.InternalIdentity; -import org.apache.cloudstack.framework.config.ConfigKey; import java.util.Map; @@ -28,9 +27,6 @@ import java.util.Map; * */ public interface DataCenter extends InfrastructureEntity, Grouping, Identity, InternalIdentity { - public static final String SystemVMUseLocalStorageCK = "system.vm.use.local.storage"; - public static final ConfigKey UseSystemVMLocalStorage = new ConfigKey(Boolean.class, SystemVMUseLocalStorageCK, "Advanced", "false", - "Indicates whether to use local storage pools or shared storage pools for system VMs.", true, ConfigKey.Scope.Zone, null); public enum NetworkType { Basic, Advanced, diff --git a/client/WEB-INF/classes/resources/messages.properties b/client/WEB-INF/classes/resources/messages.properties index 7223552961d..f5e058855dc 100644 --- a/client/WEB-INF/classes/resources/messages.properties +++ b/client/WEB-INF/classes/resources/messages.properties @@ -753,7 +753,8 @@ label.load.balancer=Load Balancer label.load.balancing.policies=Load balancing policies label.load.balancing=Load Balancing label.loading=Loading -label.local.storage.enabled=Local storage enabled +label.local.storage.enabled=Enable local storage for User VMs +label.local.storage.enabled.system.vms=Enable local storage for System VMs label.local.storage=Local Storage label.local=Local label.login=Login diff --git a/engine/schema/src/com/cloud/service/dao/ServiceOfferingDao.java b/engine/schema/src/com/cloud/service/dao/ServiceOfferingDao.java index ab818538d2a..586fcd0f797 100644 --- a/engine/schema/src/com/cloud/service/dao/ServiceOfferingDao.java +++ b/engine/schema/src/com/cloud/service/dao/ServiceOfferingDao.java @@ -16,18 +16,24 @@ // under the License. package com.cloud.service.dao; +import com.cloud.service.ServiceOfferingVO; +import com.cloud.storage.Storage; +import com.cloud.utils.db.GenericDao; +import com.cloud.vm.VirtualMachine; + import java.util.List; import java.util.Map; -import com.cloud.service.ServiceOfferingVO; -import com.cloud.utils.db.GenericDao; - /* * Data Access Object for service_offering table */ public interface ServiceOfferingDao extends GenericDao { ServiceOfferingVO findByName(String name); + List createSystemServiceOfferings(String name, String uniqueName, int cpuCount, int ramSize, int cpuSpeed, + Integer rateMbps, Integer multicastRateMbps, boolean offerHA, String displayText, Storage.ProvisioningType provisioningType, + boolean recreatable, String tags, boolean systemUse, VirtualMachine.Type vmType, boolean defaultUse); + ServiceOfferingVO persistSystemServiceOffering(ServiceOfferingVO vo); List findPublicServiceOfferings(); @@ -49,4 +55,6 @@ public interface ServiceOfferingDao extends GenericDao boolean isDynamic(long serviceOfferingId); ServiceOfferingVO getcomputeOffering(ServiceOfferingVO serviceOffering, Map customParameters); + + ServiceOfferingVO findDefaultSystemOffering(String offeringName, Boolean useLocalStorage); } diff --git a/engine/schema/src/com/cloud/service/dao/ServiceOfferingDaoImpl.java b/engine/schema/src/com/cloud/service/dao/ServiceOfferingDaoImpl.java index a3ff45ca3ad..280a4ba7c7f 100644 --- a/engine/schema/src/com/cloud/service/dao/ServiceOfferingDaoImpl.java +++ b/engine/schema/src/com/cloud/service/dao/ServiceOfferingDaoImpl.java @@ -25,6 +25,8 @@ import javax.ejb.Local; import javax.inject.Inject; import javax.persistence.EntityExistsException; +import com.cloud.storage.Storage; +import com.cloud.vm.VirtualMachine; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -110,6 +112,13 @@ public class ServiceOfferingDaoImpl extends GenericDaoBase createSystemServiceOfferings(String name, String uniqueName, int cpuCount, int ramSize, int cpuSpeed, + Integer rateMbps, Integer multicastRateMbps, boolean offerHA, String displayText, Storage.ProvisioningType provisioningType, + boolean recreatable, String tags, boolean systemUse, VirtualMachine.Type vmType, boolean defaultUse) { + List list = new ArrayList(); + ServiceOfferingVO offering = new ServiceOfferingVO(name, cpuCount, ramSize, cpuSpeed, rateMbps, multicastRateMbps, offerHA, displayText, + provisioningType, false, recreatable, tags, systemUse, vmType, defaultUse); + offering.setUniqueName(uniqueName); + offering = persistSystemServiceOffering(offering); + if (offering != null) { + list.add(offering); + } + + boolean useLocal = true; + if (offering.getUseLocalStorage()) { // if 1st one is already local then 2nd needs to be shared + useLocal = false; + } + + offering = new ServiceOfferingVO(name + (useLocal ? " - Local Storage" : ""), cpuCount, ramSize, cpuSpeed, rateMbps, multicastRateMbps, offerHA, displayText, + provisioningType, useLocal, recreatable, tags, systemUse, vmType, defaultUse); + offering.setUniqueName(uniqueName + (useLocal ? "-Local" : "")); + offering = persistSystemServiceOffering(offering); + if (offering != null) { + list.add(offering); + } + + return list; + } + + @Override + public ServiceOfferingVO findDefaultSystemOffering(String offeringName, Boolean useLocalStorage) { + String name = offeringName; + if (useLocalStorage != null && useLocalStorage.booleanValue()) { + name += "-Local"; + } + ServiceOfferingVO serviceOffering = findByName(name); + if (serviceOffering == null) { + String message = "System service offering " + name + " not found"; + s_logger.error(message); + throw new CloudRuntimeException(message); + } + return serviceOffering; + } } diff --git a/plugins/network-elements/elastic-loadbalancer/src/com/cloud/network/lb/ElasticLoadBalancerManagerImpl.java b/plugins/network-elements/elastic-loadbalancer/src/com/cloud/network/lb/ElasticLoadBalancerManagerImpl.java index d853299eefd..11da736d1d0 100644 --- a/plugins/network-elements/elastic-loadbalancer/src/com/cloud/network/lb/ElasticLoadBalancerManagerImpl.java +++ b/plugins/network-elements/elastic-loadbalancer/src/com/cloud/network/lb/ElasticLoadBalancerManagerImpl.java @@ -148,7 +148,6 @@ public class ElasticLoadBalancerManagerImpl extends ManagerBase implements Elast TrafficType _frontendTrafficType = TrafficType.Guest; Account _systemAcct; - ServiceOfferingVO _elasticLbVmOffering; ScheduledExecutorService _gcThreadPool; String _mgmtCidr; @@ -290,16 +289,19 @@ public class ElasticLoadBalancerManagerImpl extends ManagerBase implements Elast } _mgmtCidr = _configDao.getValue(Config.ManagementNetwork.key()); - boolean useLocalStorage = Boolean.parseBoolean(configs.get(DataCenter.SystemVMUseLocalStorageCK)); - _elasticLbVmRamSize = NumbersUtil.parseInt(configs.get(Config.ElasticLoadBalancerVmMemory.key()), DEFAULT_ELB_VM_RAMSIZE); _elasticLbvmCpuMHz = NumbersUtil.parseInt(configs.get(Config.ElasticLoadBalancerVmCpuMhz.key()), DEFAULT_ELB_VM_CPU_MHZ); _elasticLbvmNumCpu = NumbersUtil.parseInt(configs.get(Config.ElasticLoadBalancerVmNumVcpu.key()), 1); - _elasticLbVmOffering = new ServiceOfferingVO("System Offering For Elastic LB VM", _elasticLbvmNumCpu, - _elasticLbVmRamSize, _elasticLbvmCpuMHz, 0, 0, true, null, Storage.ProvisioningType.THIN, useLocalStorage, - true, null, true, VirtualMachine.Type.ElasticLoadBalancerVm, true); - _elasticLbVmOffering.setUniqueName(ServiceOffering.elbVmDefaultOffUniqueName); - _elasticLbVmOffering = _serviceOfferingDao.persistSystemServiceOffering(_elasticLbVmOffering); + + List offerings = _serviceOfferingDao.createSystemServiceOfferings("System Offering For Elastic LB VM", + ServiceOffering.elbVmDefaultOffUniqueName, _elasticLbvmNumCpu, _elasticLbVmRamSize, _elasticLbvmCpuMHz, 0, 0, true, null, + Storage.ProvisioningType.THIN, true, null, true, VirtualMachine.Type.ElasticLoadBalancerVm, true); + // this can sometimes happen, if DB is manually or programmatically manipulated + if (offerings == null || offerings.size() < 2) { + String msg = "Data integrity problem : System Offering For Elastic LB VM has been removed?"; + s_logger.error(msg); + throw new ConfigurationException(msg); + } String enabled = _configDao.getValue(Config.ElasticLoadBalancerEnabled.key()); _enabled = (enabled == null) ? false : Boolean.parseBoolean(enabled); @@ -322,7 +324,7 @@ public class ElasticLoadBalancerManagerImpl extends ManagerBase implements Elast _itMgr.registerGuru(VirtualMachine.Type.ElasticLoadBalancerVm, this); } - loadBalanceRuleHandler = new LoadBalanceRuleHandler(_elasticLbVmOffering, _instance, _systemAcct); + loadBalanceRuleHandler = new LoadBalanceRuleHandler(_instance, _systemAcct); return true; } diff --git a/plugins/network-elements/elastic-loadbalancer/src/com/cloud/network/lb/LoadBalanceRuleHandler.java b/plugins/network-elements/elastic-loadbalancer/src/com/cloud/network/lb/LoadBalanceRuleHandler.java index 32292cddd47..848b91a416d 100644 --- a/plugins/network-elements/elastic-loadbalancer/src/com/cloud/network/lb/LoadBalanceRuleHandler.java +++ b/plugins/network-elements/elastic-loadbalancer/src/com/cloud/network/lb/LoadBalanceRuleHandler.java @@ -27,6 +27,9 @@ import java.util.Random; import javax.inject.Inject; +import com.cloud.configuration.ConfigurationManagerImpl; +import com.cloud.offering.ServiceOffering; +import com.cloud.service.dao.ServiceOfferingDao; import org.apache.cloudstack.api.command.user.loadbalancer.CreateLoadBalancerRuleCmd; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; @@ -140,15 +143,15 @@ public class LoadBalanceRuleHandler { private PhysicalNetworkServiceProviderDao _physicalProviderDao; @Inject private VirtualRouterProviderDao _vrProviderDao; + @Inject + private ServiceOfferingDao _serviceOfferingDao; static final private String ELB_VM_NAME_PREFIX = "l"; - private final ServiceOfferingVO _elasticLbVmOffering; private final String _instance; private final Account _systemAcct; - public LoadBalanceRuleHandler(ServiceOfferingVO elasticLbVmOffering, String instance, Account systemAcct) { - this._elasticLbVmOffering = elasticLbVmOffering; + public LoadBalanceRuleHandler(String instance, Account systemAcct) { this._instance = instance; this._systemAcct = systemAcct; } @@ -272,12 +275,13 @@ public class LoadBalanceRuleHandler { throw new CloudRuntimeException("Cannot find virtual router provider " + typeString + " as service provider " + provider.getId()); } - elbVm = new DomainRouterVO(id, _elasticLbVmOffering.getId(), vrProvider.getId(), VirtualMachineName.getSystemVmName(id, _instance, ELB_VM_NAME_PREFIX), + ServiceOfferingVO elasticLbVmOffering = _serviceOfferingDao.findDefaultSystemOffering(ServiceOffering.elbVmDefaultOffUniqueName, ConfigurationManagerImpl.SystemVMUseLocalStorage.valueIn(dest.getDataCenter().getId())); + elbVm = new DomainRouterVO(id, elasticLbVmOffering.getId(), vrProvider.getId(), VirtualMachineName.getSystemVmName(id, _instance, ELB_VM_NAME_PREFIX), template.getId(), template.getHypervisorType(), template.getGuestOSId(), owner.getDomainId(), owner.getId(), false, 0, false, RedundantState.UNKNOWN, - _elasticLbVmOffering.getOfferHA(), false, VirtualMachine.Type.ElasticLoadBalancerVm, null); + elasticLbVmOffering.getOfferHA(), false, VirtualMachine.Type.ElasticLoadBalancerVm, null); elbVm.setRole(Role.LB); elbVm = _routerDao.persist(elbVm); - _itMgr.allocate(elbVm.getInstanceName(), template, _elasticLbVmOffering, networks, plan, null); + _itMgr.allocate(elbVm.getInstanceName(), template, elasticLbVmOffering, networks, plan, null); elbVm = _routerDao.findById(elbVm.getId()); //TODO: create usage stats } diff --git a/plugins/network-elements/internal-loadbalancer/src/org/apache/cloudstack/network/lb/InternalLoadBalancerVMManagerImpl.java b/plugins/network-elements/internal-loadbalancer/src/org/apache/cloudstack/network/lb/InternalLoadBalancerVMManagerImpl.java index cf7715071f3..eee1f12261a 100644 --- a/plugins/network-elements/internal-loadbalancer/src/org/apache/cloudstack/network/lb/InternalLoadBalancerVMManagerImpl.java +++ b/plugins/network-elements/internal-loadbalancer/src/org/apache/cloudstack/network/lb/InternalLoadBalancerVMManagerImpl.java @@ -27,6 +27,7 @@ import javax.ejb.Local; import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.configuration.ConfigurationManagerImpl; import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.lb.ApplicationLoadBalancerRuleVO; @@ -376,15 +377,15 @@ public class InternalLoadBalancerVMManagerImpl extends ManagerBase implements In //if offering wasn't set, try to get the default one if (_internalLbVmOfferingId == 0L) { - boolean useLocalStorage = Boolean.parseBoolean(configs.get(DataCenter.SystemVMUseLocalStorageCK)); - ServiceOfferingVO newOff = - new ServiceOfferingVO("System Offering For Internal LB VM", 1, InternalLoadBalancerVMManager.DEFAULT_INTERNALLB_VM_RAMSIZE, + List offerings = _serviceOfferingDao.createSystemServiceOfferings("System Offering For Internal LB VM", + ServiceOffering.internalLbVmDefaultOffUniqueName, 1, InternalLoadBalancerVMManager.DEFAULT_INTERNALLB_VM_RAMSIZE, InternalLoadBalancerVMManager.DEFAULT_INTERNALLB_VM_CPU_MHZ, null, null, true, null, - Storage.ProvisioningType.THIN, useLocalStorage, true, null, true, - VirtualMachine.Type.InternalLoadBalancerVm, true); - newOff.setUniqueName(ServiceOffering.internalLbVmDefaultOffUniqueName); - newOff = _serviceOfferingDao.persistSystemServiceOffering(newOff); - _internalLbVmOfferingId = newOff.getId(); + Storage.ProvisioningType.THIN, true, null, true, VirtualMachine.Type.InternalLoadBalancerVm, true); + if (offerings == null || offerings.size() < 2) { + String msg = "Data integrity problem : System Offering For Internal LB VM has been removed?"; + s_logger.error(msg); + throw new ConfigurationException(msg); + } } _itMgr.registerGuru(VirtualMachine.Type.InternalLoadBalancerVm, this); @@ -616,9 +617,14 @@ public class InternalLoadBalancerVMManagerImpl extends ManagerBase implements In } LinkedHashMap> networks = createInternalLbVmNetworks(guestNetwork, plan, requestedGuestIp); + long internalLbVmOfferingId = _internalLbVmOfferingId; + if (internalLbVmOfferingId == 0L) { + ServiceOfferingVO serviceOffering = _serviceOfferingDao.findDefaultSystemOffering(ServiceOffering.internalLbVmDefaultOffUniqueName, ConfigurationManagerImpl.SystemVMUseLocalStorage.valueIn(dest.getDataCenter().getId())); + internalLbVmOfferingId = serviceOffering.getId(); + } //Pass startVm=false as we are holding the network lock that needs to be released at the end of vm allocation DomainRouterVO internalLbVm = - deployInternalLbVm(owner, dest, plan, params, internalLbProviderId, _internalLbVmOfferingId, guestNetwork.getVpcId(), networks, false); + deployInternalLbVm(owner, dest, plan, params, internalLbProviderId, internalLbVmOfferingId, guestNetwork.getVpcId(), networks, false); if (internalLbVm != null) { _internalLbVmDao.addRouterToGuestNetwork(internalLbVm, guestNetwork); internalLbVms.add(internalLbVm); diff --git a/plugins/network-elements/internal-loadbalancer/test/org/apache/cloudstack/internallbvmmgr/InternalLBVMManagerTest.java b/plugins/network-elements/internal-loadbalancer/test/org/apache/cloudstack/internallbvmmgr/InternalLBVMManagerTest.java index 375ba5e593e..5b5d65a3cd4 100644 --- a/plugins/network-elements/internal-loadbalancer/test/org/apache/cloudstack/internallbvmmgr/InternalLBVMManagerTest.java +++ b/plugins/network-elements/internal-loadbalancer/test/org/apache/cloudstack/internallbvmmgr/InternalLBVMManagerTest.java @@ -121,7 +121,12 @@ public class InternalLBVMManagerTest extends TestCase { ServiceOfferingVO off = new ServiceOfferingVO("alena", 1, 1, 1, 1, 1, false, "alena", Storage.ProvisioningType.THIN, false, false, null, false, VirtualMachine.Type.InternalLoadBalancerVm, false); off = setId(off, 1); - Mockito.when(_svcOffDao.persistSystemServiceOffering(Matchers.any(ServiceOfferingVO.class))).thenReturn(off); + List list = new ArrayList(); + list.add(off); + list.add(off); + Mockito.when(_svcOffDao.createSystemServiceOfferings(Matchers.anyString(), Matchers.anyString(), Matchers.anyInt(), Matchers.anyInt(), Matchers.anyInt(), + Matchers.anyInt(), Matchers.anyInt(), Matchers.anyBoolean(), Matchers.anyString(), Matchers.any(Storage.ProvisioningType.class), Matchers.anyBoolean(), + Matchers.anyString(), Matchers.anyBoolean(), Matchers.any(VirtualMachine.Type.class), Matchers.anyBoolean())).thenReturn(list); ComponentContext.initComponentsLifeCycle(); diff --git a/plugins/network-elements/internal-loadbalancer/test/org/apache/cloudstack/internallbvmmgr/InternalLBVMServiceTest.java b/plugins/network-elements/internal-loadbalancer/test/org/apache/cloudstack/internallbvmmgr/InternalLBVMServiceTest.java index 11a0bed8e81..ec636fa797b 100644 --- a/plugins/network-elements/internal-loadbalancer/test/org/apache/cloudstack/internallbvmmgr/InternalLBVMServiceTest.java +++ b/plugins/network-elements/internal-loadbalancer/test/org/apache/cloudstack/internallbvmmgr/InternalLBVMServiceTest.java @@ -17,6 +17,8 @@ package org.apache.cloudstack.internallbvmmgr; import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.List; import javax.inject.Inject; @@ -91,7 +93,12 @@ public class InternalLBVMServiceTest extends TestCase { ServiceOfferingVO off = new ServiceOfferingVO("alena", 1, 1, 1, 1, 1, false, "alena", Storage.ProvisioningType.THIN, false, false, null, false, VirtualMachine.Type.InternalLoadBalancerVm, false); off = setId(off, 1); - Mockito.when(_svcOffDao.persistSystemServiceOffering(Matchers.any(ServiceOfferingVO.class))).thenReturn(off); + List list = new ArrayList(); + list.add(off); + list.add(off); + Mockito.when(_svcOffDao.createSystemServiceOfferings(Matchers.anyString(), Matchers.anyString(), Matchers.anyInt(), Matchers.anyInt(), Matchers.anyInt(), + Matchers.anyInt(), Matchers.anyInt(), Matchers.anyBoolean(), Matchers.anyString(), Matchers.any(Storage.ProvisioningType.class), Matchers.anyBoolean(), + Matchers.anyString(), Matchers.anyBoolean(), Matchers.any(VirtualMachine.Type.class), Matchers.anyBoolean())).thenReturn(list); ComponentContext.initComponentsLifeCycle(); diff --git a/server/src/com/cloud/configuration/Config.java b/server/src/com/cloud/configuration/Config.java index 0d6a4e66f4f..d4a77976c49 100755 --- a/server/src/com/cloud/configuration/Config.java +++ b/server/src/com/cloud/configuration/Config.java @@ -684,7 +684,6 @@ public enum Config { "/var/cloudstack/mnt", "The mount point on the Management Server for Secondary Storage.", null), -// UpgradeURL("Advanced", ManagementServer.class, String.class, "upgrade.url", "http://example.com:8080/client/agent/update.zip", "The upgrade URL is the URL of the management server that agents will connect to in order to automatically upgrade.", null), SystemVMAutoReserveCapacity( "Advanced", ManagementServer.class, diff --git a/server/src/com/cloud/configuration/ConfigurationManagerImpl.java b/server/src/com/cloud/configuration/ConfigurationManagerImpl.java index 89027c12401..89e0428bad9 100755 --- a/server/src/com/cloud/configuration/ConfigurationManagerImpl.java +++ b/server/src/com/cloud/configuration/ConfigurationManagerImpl.java @@ -37,6 +37,7 @@ import javax.ejb.Local; import javax.inject.Inject; import javax.naming.ConfigurationException; +import org.apache.cloudstack.framework.config.Configurable; import org.apache.log4j.Logger; import org.apache.cloudstack.acl.SecurityChecker; @@ -214,8 +215,11 @@ import com.cloud.vm.dao.NicIpAliasVO; import com.cloud.vm.dao.NicSecondaryIpDao; @Local(value = {ConfigurationManager.class, ConfigurationService.class}) -public class ConfigurationManagerImpl extends ManagerBase implements ConfigurationManager, ConfigurationService { +public class ConfigurationManagerImpl extends ManagerBase implements ConfigurationManager, ConfigurationService, Configurable { public static final Logger s_logger = Logger.getLogger(ConfigurationManagerImpl.class); + public static final String SystemVMUseLocalStorageCK = "system.vm.use.local.storage"; + public static final ConfigKey SystemVMUseLocalStorage = new ConfigKey(Boolean.class, SystemVMUseLocalStorageCK, "Advanced", "false", + "Indicates whether to use local storage pools or shared storage pools for system VMs.", true, ConfigKey.Scope.Zone, null); @Inject EntityManager _entityMgr; @@ -570,35 +574,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati } catch (Throwable e) { throw new CloudRuntimeException("Failed to update storage.network.device2 in host_details due to exception ", e); } - } else if (DataCenter.SystemVMUseLocalStorageCK.equalsIgnoreCase(name)) { - if (s_logger.isDebugEnabled()) { - s_logger.debug("Config 'system.vm.use.local.storage' changed to value:" + value + ", need to update System VM offerings"); - } - boolean useLocalStorage = Boolean.parseBoolean(_configDao.getValue(DataCenter.SystemVMUseLocalStorageCK)); - ServiceOfferingVO serviceOffering = _serviceOfferingDao.findByName(ServiceOffering.consoleProxyDefaultOffUniqueName); - if (serviceOffering != null) { - serviceOffering.setUseLocalStorage(useLocalStorage); - if (!_serviceOfferingDao.update(serviceOffering.getId(), serviceOffering)) { - throw new CloudRuntimeException("Failed to update ConsoleProxy offering's use_local_storage option to value:" + useLocalStorage); - } - } - - serviceOffering = _serviceOfferingDao.findByName(ServiceOffering.routerDefaultOffUniqueName); - if (serviceOffering != null) { - serviceOffering.setUseLocalStorage(useLocalStorage); - if (!_serviceOfferingDao.update(serviceOffering.getId(), serviceOffering)) { - throw new CloudRuntimeException("Failed to update SoftwareRouter offering's use_local_storage option to value:" + useLocalStorage); - } - } - - serviceOffering = _serviceOfferingDao.findByName(ServiceOffering.ssvmDefaultOffUniqueName); - if (serviceOffering != null) { - serviceOffering.setUseLocalStorage(useLocalStorage); - if (!_serviceOfferingDao.update(serviceOffering.getId(), serviceOffering)) { - throw new CloudRuntimeException("Failed to update SecondaryStorage offering's use_local_storage option to value:" + useLocalStorage); - } - } - }else if (Config.SecStorageSecureCopyCert.key().equalsIgnoreCase(name)) { + } else if (Config.SecStorageSecureCopyCert.key().equalsIgnoreCase(name)) { //FIXME - Ideally there should be a listener model to listen to global config changes and be able to take action gracefully. //Expire the download urls String sqlTemplate = "update template_store_ref set download_url_created=?"; @@ -5081,4 +5057,13 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati _secChecker = secChecker; } + @Override + public String getConfigComponentName() { + return ConfigurationManagerImpl.class.getSimpleName(); + } + + @Override + public ConfigKey[] getConfigKeys() { + return new ConfigKey[] {SystemVMUseLocalStorage}; + } } diff --git a/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java b/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java index 3fd35437a7f..df338c441de 100755 --- a/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java +++ b/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java @@ -30,6 +30,7 @@ import javax.ejb.Local; import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.configuration.ConfigurationManagerImpl; import org.apache.log4j.Logger; import com.google.gson.Gson; @@ -225,7 +226,6 @@ public class ConsoleProxyManagerImpl extends ManagerBase implements ConsoleProxy private int _capacityPerProxy = ConsoleProxyManager.DEFAULT_PROXY_CAPACITY; private int _standbyCapacity = ConsoleProxyManager.DEFAULT_STANDBY_CAPACITY; - private boolean _useLvm; private boolean _useStorageVm; private boolean _disableRpFilter = false; private String _instance; @@ -716,13 +716,18 @@ public class ConsoleProxyManagerImpl extends ManagerBase implements ConsoleProxy networks.put(_networkMgr.setupNetwork(systemAcct, offering, plan, null, null, false).get(0), new ArrayList()); } + ServiceOfferingVO serviceOffering = _serviceOffering; + if (serviceOffering == null) { + serviceOffering = _offeringDao.findDefaultSystemOffering(ServiceOffering.consoleProxyDefaultOffUniqueName, ConfigurationManagerImpl.SystemVMUseLocalStorage.valueIn(dataCenterId)); + } + ConsoleProxyVO proxy = - new ConsoleProxyVO(id, _serviceOffering.getId(), name, template.getId(), template.getHypervisorType(), template.getGuestOSId(), dataCenterId, - systemAcct.getDomainId(), systemAcct.getId(), 0, _serviceOffering.getOfferHA()); + new ConsoleProxyVO(id, serviceOffering.getId(), name, template.getId(), template.getHypervisorType(), template.getGuestOSId(), dataCenterId, + systemAcct.getDomainId(), systemAcct.getId(), 0, serviceOffering.getOfferHA()); proxy.setDynamicallyScalable(template.isDynamicallyScalable()); proxy = _consoleProxyDao.persist(proxy); try { - _itMgr.allocate(name, template, _serviceOffering, networks, plan, null); + _itMgr.allocate(name, template, serviceOffering, networks, plan, null); } catch (InsufficientCapacityException e) { s_logger.warn("InsufficientCapacity", e); throw new CloudRuntimeException("Insufficient capacity exception", e); @@ -951,7 +956,12 @@ public class ConsoleProxyManagerImpl extends ManagerBase implements ConsoleProxy TemplateDataStoreVO templateHostRef = _vmTemplateStoreDao.findByTemplateZoneDownloadStatus(template.getId(), dataCenterId, Status.DOWNLOADED); if (templateHostRef != null) { - List> l = _consoleProxyDao.getDatacenterStoragePoolHostInfo(dataCenterId, _useLvm); + boolean useLocalStorage = false; + Boolean useLocal = ConfigurationManagerImpl.SystemVMUseLocalStorage.valueIn(dataCenterId); + if (useLocal != null) { + useLocalStorage = useLocal; + } + List> l = _consoleProxyDao.getDatacenterStoragePoolHostInfo(dataCenterId, useLocalStorage); if (l != null && l.size() > 0 && l.get(0).second().intValue() > 0) { return true; } else { @@ -1208,11 +1218,6 @@ public class ConsoleProxyManagerImpl extends ManagerBase implements ConsoleProxy _disableRpFilter = true; } - value = configs.get(DataCenter.SystemVMUseLocalStorageCK); - if (value != null && value.equalsIgnoreCase("true")) { - _useLvm = true; - } - value = configs.get("secondary.storage.vm"); if (value != null && value.equalsIgnoreCase("true")) { _useStorageVm = true; @@ -1238,8 +1243,6 @@ public class ConsoleProxyManagerImpl extends ManagerBase implements ConsoleProxy _itMgr.registerGuru(VirtualMachine.Type.ConsoleProxy, this); - boolean useLocalStorage = Boolean.parseBoolean(configs.get(DataCenter.SystemVMUseLocalStorageCK)); - //check if there is a default service offering configured String cpvmSrvcOffIdStr = configs.get(Config.ConsoleProxyServiceOffering.key()); if (cpvmSrvcOffIdStr != null) { @@ -1259,15 +1262,13 @@ public class ConsoleProxyManagerImpl extends ManagerBase implements ConsoleProxy if (_serviceOffering == null || !_serviceOffering.getSystemUse()) { int ramSize = NumbersUtil.parseInt(_configDao.getValue("console.ram.size"), DEFAULT_PROXY_VM_RAMSIZE); int cpuFreq = NumbersUtil.parseInt(_configDao.getValue("console.cpu.mhz"), DEFAULT_PROXY_VM_CPUMHZ); - _serviceOffering = - new ServiceOfferingVO("System Offering For Console Proxy", 1, ramSize, cpuFreq, 0, 0, false, null, - Storage.ProvisioningType.THIN, useLocalStorage, true, null, true, - VirtualMachine.Type.ConsoleProxy, true); - _serviceOffering.setUniqueName(ServiceOffering.consoleProxyDefaultOffUniqueName); - _serviceOffering = _offeringDao.persistSystemServiceOffering(_serviceOffering); + + List offerings = _offeringDao.createSystemServiceOfferings("System Offering For Console Proxy", + ServiceOffering.consoleProxyDefaultOffUniqueName, 1, ramSize, cpuFreq, 0, 0, false, null, + Storage.ProvisioningType.THIN, true, null, true, VirtualMachine.Type.ConsoleProxy, true); // this can sometimes happen, if DB is manually or programmatically manipulated - if (_serviceOffering == null) { + if (offerings == null || offerings.size() < 2) { String msg = "Data integrity problem : System Offering For Console Proxy has been removed?"; s_logger.error(msg); throw new ConfigurationException(msg); diff --git a/server/src/com/cloud/deploy/DeploymentPlanningManagerImpl.java b/server/src/com/cloud/deploy/DeploymentPlanningManagerImpl.java index c290e9c7966..7637a3aa173 100755 --- a/server/src/com/cloud/deploy/DeploymentPlanningManagerImpl.java +++ b/server/src/com/cloud/deploy/DeploymentPlanningManagerImpl.java @@ -31,6 +31,7 @@ import javax.ejb.Local; import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.configuration.ConfigurationManagerImpl; import com.cloud.utils.fsm.StateMachine2; import org.apache.log4j.Logger; @@ -46,8 +47,6 @@ import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.engine.subsystem.api.storage.StoragePoolAllocator; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; -import org.apache.cloudstack.framework.config.ConfigKey; -import org.apache.cloudstack.framework.config.Configurable; import org.apache.cloudstack.framework.messagebus.MessageBus; import org.apache.cloudstack.framework.messagebus.MessageSubscriber; import org.apache.cloudstack.managed.context.ManagedContextTimerTask; @@ -134,7 +133,7 @@ import com.cloud.vm.dao.VMInstanceDao; @Local(value = {DeploymentPlanningManager.class}) public class DeploymentPlanningManagerImpl extends ManagerBase implements DeploymentPlanningManager, Manager, Listener, -StateListener, Configurable { +StateListener { private static final Logger s_logger = Logger.getLogger(DeploymentPlanningManagerImpl.class); @Inject @@ -755,16 +754,6 @@ StateListener, Configurable { return false; } - @Override - public String getConfigComponentName() { - return DeploymentPlanningManagerImpl.class.getSimpleName(); - } - - @Override - public ConfigKey[] getConfigKeys() { - return new ConfigKey[] {DataCenter.UseSystemVMLocalStorage}; - } - class HostReservationReleaseChecker extends ManagedContextTimerTask { @Override protected void runInContext() { @@ -1294,21 +1283,13 @@ StateListener, Configurable { boolean useLocalStorage = false; if (vmProfile.getType() != VirtualMachine.Type.User) { DataCenterVO zone = _dcDao.findById(plan.getDataCenterId()); - // It should not happen to have a "null" zone here. There can be NO instance if there is NO zone, - // so this part of the code would never be reached if no zone has been created. - // Added the check and the comment just to make it clear. - boolean zoneUsesLocalStorage = zone != null ? zone.isLocalStorageEnabled() : false; - boolean ssvmUseLocalStorage = DataCenter.UseSystemVMLocalStorage.value(); - if (zone != null) { - ssvmUseLocalStorage = DataCenter.UseSystemVMLocalStorage.valueIn(plan.getDataCenterId()); - } - s_logger.debug("Checking if we need local storage for systemvms is needed for zone id=" + plan.getDataCenterId() + " with system.vm.use.local.storage=" + ssvmUseLocalStorage); - // Local storage is used for the NON User VMs if, and only if, the Zone is marked to use local storage AND - // the global settings (ssvmUseLocalStorage) is set to true. Otherwise, the global settings won't be applied. - if (ssvmUseLocalStorage && zoneUsesLocalStorage) { - useLocalStorage = true; - s_logger.debug("SystemVMs will use local storage for zone id=" + plan.getDataCenterId()); + assert (zone != null) : "Invalid zone in deployment plan"; + Boolean useLocalStorageForSystemVM = ConfigurationManagerImpl.SystemVMUseLocalStorage.valueIn(zone.getId()); + if (useLocalStorageForSystemVM != null) { + useLocalStorage = useLocalStorageForSystemVM.booleanValue(); + s_logger.debug("System VMs will use " + (useLocalStorage ? "local" : "shared") + " storage for zone id=" + plan.getDataCenterId()); } + } else { useLocalStorage = diskOffering.getUseLocalStorage(); diff --git a/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java b/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java index e0d1edb5f89..73aa94c83df 100755 --- a/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java +++ b/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java @@ -72,6 +72,7 @@ import com.cloud.cluster.ManagementServerHostVO; import com.cloud.cluster.dao.ManagementServerHostDao; import com.cloud.configuration.Config; import com.cloud.configuration.ConfigurationManager; +import com.cloud.configuration.ConfigurationManagerImpl; import com.cloud.configuration.ZoneConfig; import com.cloud.dc.ClusterVO; import com.cloud.dc.DataCenter; @@ -737,14 +738,11 @@ VirtualMachineGuru, Listener, Configurable, StateListener offerings = _serviceOfferingDao.createSystemServiceOfferings("System Offering For Software Router", + ServiceOffering.routerDefaultOffUniqueName, 1, _routerRamSize, _routerCpuMHz, null, + null, true, null, ProvisioningType.THIN, true, null, true, VirtualMachine.Type.DomainRouter, true); // this can sometimes happen, if DB is manually or programmatically manipulated - if (_offering == null) { + if (offerings == null || offerings.size() < 2) { final String msg = "Data integrity problem : System Offering For Software router VM has been removed?"; s_logger.error(msg); throw new ConfigurationException(msg); @@ -1676,7 +1674,8 @@ VirtualMachineGuru, Listener, Configurable, StateListener pNtwks = _pNtwkDao.listByZone(vpc.getZoneId()); diff --git a/server/src/com/cloud/storage/StorageManagerImpl.java b/server/src/com/cloud/storage/StorageManagerImpl.java index 9c442acd545..aeb0bb5f59f 100755 --- a/server/src/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/com/cloud/storage/StorageManagerImpl.java @@ -41,6 +41,7 @@ import javax.ejb.Local; import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.configuration.ConfigurationManagerImpl; import com.cloud.hypervisor.Hypervisor; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -536,11 +537,20 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C @DB @Override public DataStore createLocalStorage(Host host, StoragePoolInfo pInfo) throws ConnectionException { - DataCenterVO dc = _dcDao.findById(host.getDataCenterId()); - if (dc == null || !dc.isLocalStorageEnabled()) { + if (dc == null) { return null; } + + boolean useLocalStorageForSystemVM = false; + Boolean isLocal = ConfigurationManagerImpl.SystemVMUseLocalStorage.valueIn(dc.getId()); + if (isLocal != null) { + useLocalStorageForSystemVM = isLocal.booleanValue(); + } + if (!(dc.isLocalStorageEnabled() || useLocalStorageForSystemVM)) { + return null; + } + DataStore store; try { String hostAddress = pInfo.getHost(); diff --git a/server/test/com/cloud/network/element/VirtualRouterElementTest.java b/server/test/com/cloud/network/element/VirtualRouterElementTest.java index d6eb24fb395..0f79f56f280 100644 --- a/server/test/com/cloud/network/element/VirtualRouterElementTest.java +++ b/server/test/com/cloud/network/element/VirtualRouterElementTest.java @@ -31,6 +31,7 @@ import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.InjectMocks; +import org.mockito.Matchers; import org.mockito.Mock; import org.mockito.runners.MockitoJUnitRunner; @@ -268,6 +269,8 @@ public class VirtualRouterElementTest { VirtualMachine.Type.DomainRouter, /* defaultUse */ false); when(_serviceOfferingDao.findById(0L)).thenReturn(svcoff); + when(_serviceOfferingDao.findByName(Matchers.anyString())).thenReturn(svcoff); + when(_serviceOfferingDao.findDefaultSystemOffering(Matchers.anyString(), Matchers.anyBoolean())).thenReturn(svcoff); DomainRouterVO router = new DomainRouterVO(/* id */ 1L, /* serviceOfferingId */ 1L, /* elementId */ 0L, diff --git a/services/secondary-storage/controller/src/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java b/services/secondary-storage/controller/src/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java index 9d89b07cca4..7b6e8680519 100755 --- a/services/secondary-storage/controller/src/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java +++ b/services/secondary-storage/controller/src/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java @@ -31,6 +31,7 @@ import javax.ejb.Local; import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.configuration.ConfigurationManagerImpl; import org.apache.cloudstack.config.ApiServiceConfiguration; import org.apache.cloudstack.context.CallContext; @@ -242,7 +243,6 @@ public class SecondaryStorageManagerImpl extends ManagerBase implements Secondar private int _secStorageVmMtuSize; private String _instance; - private boolean _useLocalStorage; private boolean _useSSlCopy; private String _httpProxy; private String _allowedInternalSites; @@ -573,13 +573,17 @@ public class SecondaryStorageManagerImpl extends ManagerBase implements Secondar throw new CloudRuntimeException("Not able to find the System templates or not downloaded in zone " + dataCenterId); } + ServiceOfferingVO serviceOffering = _serviceOffering; + if (serviceOffering == null) { + serviceOffering = _offeringDao.findDefaultSystemOffering(ServiceOffering.ssvmDefaultOffUniqueName, ConfigurationManagerImpl.SystemVMUseLocalStorage.valueIn(dataCenterId)); + } SecondaryStorageVmVO secStorageVm = - new SecondaryStorageVmVO(id, _serviceOffering.getId(), name, template.getId(), template.getHypervisorType(), template.getGuestOSId(), dataCenterId, - systemAcct.getDomainId(), systemAcct.getId(), role, _serviceOffering.getOfferHA()); + new SecondaryStorageVmVO(id, serviceOffering.getId(), name, template.getId(), template.getHypervisorType(), template.getGuestOSId(), dataCenterId, + systemAcct.getDomainId(), systemAcct.getId(), role, serviceOffering.getOfferHA()); secStorageVm.setDynamicallyScalable(template.isDynamicallyScalable()); secStorageVm = _secStorageVmDao.persist(secStorageVm); try { - _itMgr.allocate(name, template, _serviceOffering, networks, plan, null); + _itMgr.allocate(name, template, serviceOffering, networks, plan, null); secStorageVm = _secStorageVmDao.findById(secStorageVm.getId()); } catch (InsufficientCapacityException e) { s_logger.warn("InsufficientCapacity", e); @@ -759,14 +763,20 @@ public class SecondaryStorageManagerImpl extends ManagerBase implements Secondar return false; } - List> l = _storagePoolHostDao.getDatacenterStoragePoolHostInfo(dataCenterId, !_useLocalStorage); + boolean useLocalStorage = false; + Boolean useLocal = ConfigurationManagerImpl.SystemVMUseLocalStorage.valueIn(dataCenterId); + if (useLocal != null) { + useLocalStorage = useLocal.booleanValue(); + } + List> l = _storagePoolHostDao.getDatacenterStoragePoolHostInfo(dataCenterId, !useLocalStorage); + if (l != null && l.size() > 0 && l.get(0).second().intValue() > 0) { return true; } else { if (s_logger.isDebugEnabled()) { s_logger.debug("Primary storage is not ready, wait until it is ready to launch secondary storage vm. dcId: " + dataCenterId + - " system.vm.use.local.storage: " + _useLocalStorage + - "If you want to use local storage to start ssvm, need to set system.vm.use.local.storage to true"); + ", " + ConfigurationManagerImpl.SystemVMUseLocalStorage.key() + ": " + useLocalStorage + ". " + + "If you want to use local storage to start SSVM, need to set " + ConfigurationManagerImpl.SystemVMUseLocalStorage.key() + " to true"); } } @@ -871,15 +881,12 @@ public class SecondaryStorageManagerImpl extends ManagerBase implements Secondar if(_serviceOffering == null || !_serviceOffering.getSystemUse()){ int ramSize = NumbersUtil.parseInt(_configDao.getValue("ssvm.ram.size"), DEFAULT_SS_VM_RAMSIZE); int cpuFreq = NumbersUtil.parseInt(_configDao.getValue("ssvm.cpu.mhz"), DEFAULT_SS_VM_CPUMHZ); - _useLocalStorage = Boolean.parseBoolean(configs.get(DataCenter.SystemVMUseLocalStorageCK)); - _serviceOffering = - new ServiceOfferingVO("System Offering For Secondary Storage VM", 1, ramSize, cpuFreq, null, null, false, null, - Storage.ProvisioningType.THIN, _useLocalStorage, true, null, true, VirtualMachine.Type.SecondaryStorageVm, true); - _serviceOffering.setUniqueName(ServiceOffering.ssvmDefaultOffUniqueName); - _serviceOffering = _offeringDao.persistSystemServiceOffering(_serviceOffering); + List offerings = _offeringDao.createSystemServiceOfferings("System Offering For Secondary Storage VM", + ServiceOffering.ssvmDefaultOffUniqueName, 1, ramSize, cpuFreq, null, null, false, null, + Storage.ProvisioningType.THIN, true, null, true, VirtualMachine.Type.SecondaryStorageVm, true); // this can sometimes happen, if DB is manually or programmatically manipulated - if (_serviceOffering == null) { + if (offerings == null || offerings.size() < 2) { String msg = "Data integrity problem : System Offering For Secondary Storage VM has been removed?"; s_logger.error(msg); throw new ConfigurationException(msg); diff --git a/test/integration/component/maint/test_zone_level_local_storage_setting.py b/test/integration/component/maint/test_zone_level_local_storage_setting.py new file mode 100644 index 00000000000..466f1f9fd42 --- /dev/null +++ b/test/integration/component/maint/test_zone_level_local_storage_setting.py @@ -0,0 +1,734 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Test cases for zone level settings "system.vm.use.local.storage" +""" +# Import Local Modules +from marvin.cloudstackTestCase import * +from marvin.lib.utils import * +from marvin.lib.base import * +from marvin.lib.common import * +from marvin.codes import FAILED, PASS +from requests.exceptions import ConnectionError + +import time +from nose.plugins.attrib import attr +from ddt import ddt, data + + +def destroy_systemvm(self, type): + """ + Destroy system vms + #1-List system vms for current zone + #2-Destroy system vm + #3-Check if system vm came up after destroy + #4-check system vm storage type in disk offering + """ + list_response = list_ssvms( + self.apiclient, + systemvmtype=type, + zoneid=self.zone.id + ) + + self.assertEqual( + validateList(list_response)[0], + PASS, + "Check List ssvm response for %s" % + type) + + response = list_response[0] + self.debug("Destroying CPVM: %s" % response.id) + cmd = destroySystemVm.destroySystemVmCmd() + cmd.id = response.id + self.apiclient.destroySystemVm(cmd) + + timeout = self.testdata["timeout"] + while True: + time.sleep(self.testdata["sleep"]) + list_response = list_ssvms( + self.apiclient, + systemvmtype=type, + zoneid=self.zone.id + ) + if validateList(list_response)[0] == PASS: + if list_response[0].state == 'Running': + break + if timeout == 0: + raise Exception("List %s call failed!" % type) + timeout = timeout - 1 + + +def storage_check(self, type, value): + """test if system vms are using local or shared storage + #1-Get zone id from db using self.zone.id + #2-Get service offering id from vm_instance table for running system vms + #3-Get use_local_storage value from disk_offering table + #4-Verify storage type""" + query_zone_id = self.dbclient.execute( + "select id from data_center where uuid= '%s';" % self.zone.id + ) + query_so_id = self.dbclient.execute( + "select service_offering_id from vm_instance where type='%s'and " + "state='Running' and data_center_id= '%s';" % + (type, query_zone_id[0][0])) + + query_disk_offering = self.dbclient.execute( + "select use_local_storage from disk_offering where id= '%s';" % + query_so_id[0][0]) + + if value == 1: + self.assertEqual(query_disk_offering[0][0], + 1, + "system vm is not using local storage" + ) + elif value == 0: + self.assertEqual(query_disk_offering[0][0], + 0, + "system vm is not using shared storage" + ) + else: + # evil ValueError that doesn't tell you what the wrong value was + raise ValueError + + +def create_system_so(self, offering_type, storage_type): + """Create system offerings """ + self.testdata["service_offerings"]["issystem"] = "true" + self.testdata["service_offerings"]["systemvmtype"] = offering_type + self.testdata["service_offerings"]["storagetype"] = storage_type + + service_offering = ServiceOffering.create( + self.apiclient, + self.testdata["service_offerings"] + ) + + if service_offering is None: + raise Exception("service offering not created") + + list_service_response = list_service_offering( + self.apiclient, + id=service_offering.id, + issystem='true' + ) + self.assertEqual( + validateList(list_service_response)[0], + PASS, + "Check List srvice offering response for %s" % + type) + + self.debug( + "Created service offering with ID: %s" % + service_offering.id) + + self.assertEqual( + list_service_response[0].cpunumber, + self.testdata["service_offerings"]["cpunumber"], + "Check server id in createServiceOffering" + ) + self.assertEqual( + list_service_response[0].cpuspeed, + self.testdata["service_offerings"]["cpuspeed"], + "Check cpuspeed in createServiceOffering" + ) + self.assertEqual( + list_service_response[0].displaytext, + self.testdata["service_offerings"]["displaytext"], + "Check server displaytext in createServiceOfferings" + ) + self.assertEqual( + list_service_response[0].memory, + self.testdata["service_offerings"]["memory"], + "Check memory in createServiceOffering" + ) + self.assertEqual( + list_service_response[0].name, + self.testdata["service_offerings"]["name"], + "Check name in createServiceOffering" + ) + self.assertEqual( + list_service_response[0].storagetype, + self.testdata["service_offerings"]["storagetype"], + "Check storagetype in createServiceOffering" + ) + self._cleanup.append(service_offering) + return service_offering.id + + +def restart_ms(self): + """Restart MS + #1-ssh into m/c running MS + #2-restart ms + #3-verify the response + #4-loop unitl you get list_zone api answer """ + sshClient = SshClient( + self.mgtSvrDetails["mgtSvrIp"], + 22, + self.mgtSvrDetails["user"], + self.mgtSvrDetails["passwd"] + ) + command = "service cloudstack-management restart" + ms_restart_response = sshClient.execute(command) + self.assertEqual( + validateList(ms_restart_response)[0], + PASS, + "Check the MS restart response") + self.assertEqual( + ms_restart_response[0], + 'Stopping cloudstack-management:[ OK ]', + "MS i not stopped" + ) + self.assertEqual( + ms_restart_response[1], + 'Starting cloudstack-management: [ OK ]', + "MS not started" + ) + timeout = self.testdata["timeout"] + while True: + time.sleep(self.testdata["sleep"]) + try: + list_response = Zone.list( + self.apiclient + ) + if validateList(list_response)[0] == PASS: + break + except ConnectionError as e: + self.debug("list zone response is not available due to %s" % e) + + if timeout == 0: + raise Exception("Ms is not comming up !") + timeout = timeout - 1 + + +def update_global_settings(self, value, name, zoneid=None): + """Update Gloabal/zonelevel settings and verify + #1-Update configuration + #2-Restart ms if zone id is None""" + Configurations.update(self.apiclient, + name=name, + zoneid=zoneid, + value=value + ) + if zoneid is None: + restart_ms(self) + + list_conf = Configurations.list(self.apiclient, + name=name, + zoneid=zoneid) + + self.assertEqual( + validateList(list_conf)[0], + PASS, + "Check List configuration response for %s" % name) + + self.assertEqual( + str(list_conf[0].value), + str(value), + "Check if configuration values are equal" + ) + + +def str_to_bool(s): + """Converts str "True/False to Boolean TRUE/FALSE""" + if s == 'true': + return True + elif s == 'false': + return False + else: + # evil ValueError that doesn't tell you what the wrong value was + raise ValueError + + +@ddt +class TestSystemVmLocalStorage(cloudstackTestCase): + + @classmethod + def setUpClass(cls): + testClient = super(TestSystemVmLocalStorage, cls).getClsTestClient() + cls.apiclient = testClient.getApiClient() + cls.dbclient = cls.testClient.getDbConnection() + cls.mgtSvrDetails = cls.config.__dict__["mgtSvr"][0].__dict__ + cls.testdata = testClient.getParsedTestDataConfig() + # Get Zone, and template Domain + cls.domain = get_domain(cls.apiclient) + cls.zone = get_zone(cls.apiclient) + cls.testdata["mode"] = cls.zone.networktype + cls.hypervisor = testClient.getHypervisorInfo() + cls._cleanup = [] + + list_local_storage_pool = StoragePool.list( + cls.apiclient, + scope='HOST', + zoneid=cls.zone.id, + + ) + + if list_local_storage_pool is None: + + Configurations.update( + cls.apiclient, + value='true', + name='system.vm.use.local.storage', + zoneid=cls.zone.id + ) + + # Restart MS + sshClient = SshClient( + cls.mgtSvrDetails["mgtSvrIp"], + 22, + cls.mgtSvrDetails["user"], + cls.mgtSvrDetails["passwd"] + ) + command = "service cloudstack-management restart" + ms_restart_response = sshClient.execute(command) + + if validateList(ms_restart_response)[0] != PASS: + raise Exception("Check the MS restart response") + if ms_restart_response[ + 0] != 'Stopping cloudstack-management:[ OK ]': + raise Exception("MS i not stopped") + + if ms_restart_response[ + 1] != 'Starting cloudstack-management: [ OK ]': + raise Exception("MS not started") + + timeout = cls.testdata["timeout"] + while True: + # time.sleep(cls.testdata["sleep"]) + try: + list_response = Zone.list( + cls.apiclient + ) + if validateList(list_response)[0] == PASS: + break + except ConnectionError as e: + cls.debug( + "list zone response is not available due to %s" % + e) + + if timeout == 0: + raise Exception("Ms is not comming up !") + + time.sleep(cls.testdata["sleep"]) + timeout = timeout - 1 + + list_local_storage_pool = StoragePool.list( + cls.apiclient, + scope='HOST', + zoneid=cls.zone.id, + + ) + if list_local_storage_pool is None: + raise Exception("Could not discover local storage pool") + + try: + cls.account = Account.create(cls.apiclient, + cls.testdata["account"], + domainid=cls.domain.id + ) + cls._cleanup.append(cls.account) + + except Exception as e: + cls.tearDownClass() + raise e + + return + + @classmethod + def tearDownClass(cls): + try: + cleanup_resources(cls.apiclient, cls._cleanup) + except Exception as e: + raise Exception("Warning:Exception during cleanup: %s" % e) + + @attr(tags=["advanced", "basic"]) + @data( + 'consoleproxy', + 'secondarystoragevm', + 'domainrouter', + 'internalloadbalancervm') + def test_01_list_system_offerngs(self, value): + """List service offerings for systemvms and verify there should be two + (local and shared) SO for each system vm""" + + list_custom_so = ServiceOffering.list(self.apiclient, + issystem='true', + listall='true', + systemvmtype=value + ) + + self.assertEqual( + validateList(list_custom_so)[0], + PASS, + "Check List service offerings response for %s" % + value) + + local_custom_so = [] + for item in list_custom_so: + if(str(item.defaultuse) == 'True'): + local_custom_so.append(item.storagetype) + + self.assertEqual( + len(local_custom_so), + 2, + "Check default system offering for system vm type %s" % value) + if 'local' in local_custom_so and 'shared' in local_custom_so: + self.debug( + "there are exactly to Service offerings{share,local} are " + "there for system vm %s" % + value) + else: + raise Exception( + "check local and shared service offerings for %s" % + value) + + @attr(tags=["advanced", "basic"]) + @data('consoleproxy', 'secondarystoragevm') + def test_02_system_vm_storage(self, value): + """ Check if system vms are honouring zone level setting + system.vm.use.local.storage + 1-List zone level config + 2-update the zone level config with service offering uuid + 3-destroy system vms + 4-check used storage by system vms + """ + # 1 List zone level config + if value == "consoleproxy": + update_global_settings( + self, + value=None, + name="consoleproxy.service.offering") + + if value == "secondarystoragevm": + update_global_settings( + self, + value=None, + name="secstorage.service.offering") + + list_conf = Configurations.list(self.apiclient, + name="system.vm.use.local.storage", + zoneid=self.zone.id) + self.assertEqual( + validateList(list_conf)[0], + PASS, + "Check List configuration response for " + "system.vm.use.local.storage") + + val = str_to_bool(list_conf[0].value) + # 2 update the zone level config with service offering uuid + update_global_settings(self, + value=((str(not(val)).lower())), + name='system.vm.use.local.storage', + zoneid=self.zone.id) + + # 3,4 for cpvm + destroy_systemvm(self, value) + storage_check(self, value, int(not(val))) + + # 2 update the zone level config with service offering uuid + update_global_settings( + self, + value=( + str(val).lower()), + name='system.vm.use.local.storage', + zoneid=self.zone.id) + # 3,4 for cpvm + destroy_systemvm(self, value) + storage_check(self, value, int(val)) + + # 1 List zone level config + if value == "consoleproxy": + update_global_settings( + self, + value=None, + name="consoleproxy.service.offering") + + if value == "secondarystoragevm": + update_global_settings( + self, + value=None, + name="secstorage.service.offering") + + @attr(tags=["advanced", "basic"]) + @data('consoleproxy', 'secondarystoragevm') + def test_03_custom_so(self, value): + """ + update global setting with system offering and check if it is being + honoured + 1-update zone level settings "system.vm.use.local.storage={true,false}} + to use local storage + 2-create system offerings{shared,local} + 3-update global settings with system offering uuid and restart ms + 4-destroy system vms + 5-Check if new system vms are using offering updated in global + settings + """ + + # 1-update zone level settings "system.vm.use.local.storage" + # to use local storage + update_global_settings( + self, + value='true', + name='system.vm.use.local.storage', + zoneid=self.zone.id) + # 2-create system offerings + + created_so_id = create_system_so(self, value, "shared") + + if value == "consoleproxy": + name = "consoleproxy.service.offering" + elif value == 'secondarystoragevm': + name = 'secstorage.service.offering' + else: + raise Exception( + "type paramter is not correct it should be system vm " + "type{console proxy,secsroragevm}") + + # 3-update global settings with system offering uuid + update_global_settings(self, value=created_so_id, name=name) + + # 4-destroy system vms + destroy_systemvm(self, value) + + # 5-Check if new system vms are using offering updated in global + # settings + + query_zone_id = self.dbclient.execute( + "select id from data_center where uuid= '%s';" % self.zone.id + ) + query_so_id = self.dbclient.execute( + "select service_offering_id from vm_instance where type='%s'and " + "state='Running' and data_center_id= '%s';" % + (value, query_zone_id[0][0])) + query_disk_offering = self.dbclient.execute( + "select uuid from disk_offering where id= '%s';" % + query_so_id[0][0]) + + self.assertEqual( + created_so_id, + query_disk_offering[0][0], + "system vms are not using service offering mentioned in " + "global settings") + + # 6-repeate 1 with system.vm.use.local.storage=false + update_global_settings( + self, + value='false', + name='system.vm.use.local.storage', + zoneid=self.zone.id) + # 7-repeate 2 with storage type local + created_so_id = create_system_so(self, value, "local") + # 8-repeate 3 + update_global_settings(self, value=created_so_id, name=name) + + # 9-repeate 4 + destroy_systemvm(self, value) + # repeate 5 + query_zone_id = self.dbclient.execute( + "select id from data_center where uuid= '%s';" % self.zone.id + ) + query_so_id = self.dbclient.execute( + "select service_offering_id from vm_instance where type='%s'and " + "state='Running' and data_center_id= '%s';" % + (value, query_zone_id[0][0])) + query_disk_offering = self.dbclient.execute( + "select uuid from disk_offering where id= '%s';" % + query_so_id[0][0]) + + self.assertEqual( + created_so_id, + query_disk_offering[0][0], + "system vms are not using service offering mentioned in" + " global settings") + + @attr(tags=["advanced"]) + def test_04_router_vms(self): + """ Check if router vm is honouring zone level setting + system.vm.use.local.storage""" + + # 1-list configurations + list_conf = Configurations.list(self.apiclient, + name="system.vm.use.local.storage", + zoneid=self.zone.id) + self.assertEqual( + validateList(list_conf)[0], + PASS, + "Check List configuration response for " + "system.vm.use.local.storage") + + # 2-create network offering + self.network_offering = NetworkOffering.create( + self.apiclient, + self.testdata["network_offering"], + ispersistent='true' + ) + + # 3-list netwrok offerings + list_nw_of = NetworkOffering.list(self.apiclient, + id=self.network_offering.id) + self.assertEqual( + validateList(list_nw_of)[0], + PASS, + "Check the list network response" + ) + self.assertEqual( + str(list_nw_of[0].id), + str(self.network_offering.id), + "Check the created network offering id and " + "listed network offering id" + ) + self._cleanup.append(self.network_offering) + + # 4-Enable network offering + self.network_offering.update(self.apiclient, state='Enabled') + + # 5-List network offering + list_nw_of1 = NetworkOffering.list(self.apiclient, + id=self.network_offering.id) + self.assertEqual( + validateList(list_nw_of1)[0], + PASS, + "Check the list network response" + ) + self.assertEqual( + str(list_nw_of1[0].state), + "Enabled", + "Check the created network state" + ) + + # 6-crete network using network offering + self.network = Network.create( + self.apiclient, + self.testdata["network"], + networkofferingid=self.network_offering.id, + zoneid=self.zone.id, + accountid=self.account.name, + domainid=self.account.domainid + ) + # 7-List network + list_network = Network.list(self.apiclient, + accountid=self.account.name, + domainid=self.account.domainid, + id=self.network.id) + self.assertEqual(validateList(list_network)[0], + PASS, + "check list netwok response ") + self.assertEqual( + list_network[0].id, + self.network.id, + "List network id %s and created network id %s does not match" % + (list_network[0].id, + self.network.id)) + + # 8-List router + list_router = Router.list(self.apiclient, + networkid=self.network.id, + accountid=self.account.name, + domainid=self.account.domainid) + + self.assertEqual( + validateList(list_router)[0], + PASS, + "check list router response") + + # 9-List service offerings + list_so = ServiceOffering.list(self.apiclient, + issystem='true', + id=list_router[0].serviceofferingid + ) + self.assertEqual( + validateList(list_so)[0], + PASS, + "check list service offering response") + if list_conf[0].value == 'true': + storage_type = 'local' + value1 = 'false' + elif list_conf[0].value == 'false': + storage_type = 'shared' + value1 = 'true' + else: + raise Exception("check list_conf[0].value") + self.assertEqual( + list_so[0].storagetype, + storage_type, + "Check VR storage type and zone level settig" + ) + + # 10-Update zone level setting + update_global_settings( + self, + value=value1, + name="system.vm.use.local.storage", + zoneid=self.zone.id) + + # 11-List configurations + list_conf1 = Configurations.list(self.apiclient, + name="system.vm.use.local.storage", + zoneid=self.zone.id) + self.assertEqual( + validateList(list_conf1)[0], + PASS, + "Check List configuration response for " + "system.vm.use.local.storage") + + self.assertEqual( + list_conf1[0].value, + value1, + "Check the system.vm.use.local.storage value" + ) + self.network.restart(self.apiclient, + cleanup='true' + ) + # 12-List network + list_network1 = Network.list(self.apiclient, + accountid=self.account.name, + domainid=self.account.domainid, + id=self.network.id) + self.assertEqual(validateList(list_network1)[0], + PASS, + "check list netwok response ") + + # 13-list VR + list_router1 = Router.list(self.apiclient, + networkid=list_network1[0].id, + accountid=self.account.name, + domainid=self.account.domainid) + self.assertEqual( + validateList(list_router1)[0], + PASS, + "check list router response" + ) + # 14-list service offerings + list_so1 = ServiceOffering.list(self.apiclient, + issystem='true', + id=list_router1[0].serviceofferingid + ) + self.assertEqual( + validateList(list_so1)[0], + PASS, + "check list service offering response" + ) + if list_conf1[0].value == 'true': + storage_type1 = 'local' + elif list_conf1[0].value == 'false': + storage_type1 = 'shared' + else: + raise Exception("check list_conf[0].value") + self.assertEqual( + list_so1[0].storagetype, + storage_type1, + "Check VR storage type and zone level settings" + ) diff --git a/ui/dictionary2.jsp b/ui/dictionary2.jsp index 862468e3df5..20f698f9876 100644 --- a/ui/dictionary2.jsp +++ b/ui/dictionary2.jsp @@ -1014,6 +1014,7 @@ $.extend(dictionary, { 'state.detached': '', 'label.na': '', 'label.added.network.offering': '', -'label.no': '' +'label.no': '', +'label.local.storage.enabled.system.vms': '' }); diff --git a/ui/scripts/zoneWizard.js b/ui/scripts/zoneWizard.js index 700904d8a50..5f898af0fa0 100755 --- a/ui/scripts/zoneWizard.js +++ b/ui/scripts/zoneWizard.js @@ -379,7 +379,10 @@ }, addPrimaryStorage: function(args) { - return args.data.localstorageenabled != 'on'; + if(args.data.localstorageenabled == 'on' && args.data.localstorageenabledforsystemvm == 'on') { + return false; //skip step only when both localstorage and localstorage for system vm are checked + } + return true; } }, @@ -665,23 +668,13 @@ label: 'label.local.storage.enabled', isBoolean: true, onChange: function(args) { - var $checkbox = args.$checkbox; + } + }, - if ($checkbox.is(':checked')) { - cloudStack.dialog.confirm({ - message: 'message.zoneWizard.enable.local.storage', - action: function() { - $checkbox.attr('checked', true); - }, - cancelAction: function() { - $checkbox.attr('checked', false); - } - }); - - return false; - } - - return true; + localstorageenabledforsystemvm: { + label: 'label.local.storage.enabled.system.vms', + isBoolean: true, + onChange: function(args) { } } } @@ -2242,9 +2235,9 @@ } }, - action: function(args) { - var $wizard = args.wizard; - + action: function(args) { + var $wizard = args.wizard; + var formData = args.data; var advZoneConfiguredVirtualRouterCount = 0; //for multiple physical networks in advanced zone. Each physical network has 2 virtual routers: regular one and VPC one. var success = args.response.success; @@ -4419,29 +4412,50 @@ }); } - $.ajax({ - url: createURL("addHost"), - type: "POST", - data: data, - success: function(json) { - stepFns.addPrimaryStorage({ - data: $.extend(args.data, { - returnedHost: json.addhostresponse.host[0] - }) - }); - }, - error: function(XMLHttpResponse) { - var errorMsg = parseXMLHttpResponse(XMLHttpResponse); - error('addHost', errorMsg, { - fn: 'addHost', - args: args - }); - } - }); + var addHostAjax = function() { + $.ajax({ + url: createURL("addHost"), + type: "POST", + data: data, + success: function(json) { + stepFns.addPrimaryStorage({ + data: $.extend(args.data, { + returnedHost: json.addhostresponse.host[0] + }) + }); + }, + error: function(XMLHttpResponse) { + var errorMsg = parseXMLHttpResponse(XMLHttpResponse); + error('addHost', errorMsg, { + fn: 'addHost', + args: args + }); + } + }); + }; + + if(args.data.zone.localstorageenabledforsystemvm == 'on') { + $.ajax({ + url: createURL("updateConfiguration&name=system.vm.use.local.storage&value=true&zoneid=" + args.data.returnedZone.id), + dataType: "json", + success: function(json) { + addHostAjax(); + }, + error: function(XMLHttpResponse) { + var errorMsg = parseXMLHttpResponse(XMLHttpResponse); + error('addHost', errorMsg, { + fn: 'addHost', + args: args + }); + } + }); + } else { + addHostAjax(); + } }, addPrimaryStorage: function(args) { - if (args.data.zone.localstorageenabled == 'on') { //use local storage, don't need primary storage. So, skip this step. + if (args.data.zone.localstorageenabled == 'on' && args.data.zone.localstorageenabledforsystemvm == 'on') { //use local storage, don't need primary storage. So, skip stepFns.addSecondaryStorage({ data: args.data });