From e5c4bf4e28734633683fd1a3bce4f01284fec037 Mon Sep 17 00:00:00 2001 From: Chiradeep Vittal Date: Wed, 3 Aug 2011 18:55:01 -0700 Subject: [PATCH] bug 10659: manage elb vms independently of virtualnetworkappliance. --- .../lb/ElasticLoadBalancerManagerImpl.java | 208 ++++++++++++++++-- .../network/lb/dao/ElasticLbVmMapDao.java | 2 + .../network/lb/dao/ElasticLbVmMapDaoImpl.java | 100 +++++---- .../cloud/network/dao/ElbVmMapDaoTest.java | 12 + 4 files changed, 260 insertions(+), 62 deletions(-) diff --git a/server/src/com/cloud/network/lb/ElasticLoadBalancerManagerImpl.java b/server/src/com/cloud/network/lb/ElasticLoadBalancerManagerImpl.java index a18873d5d54..5f395a36bd3 100644 --- a/server/src/com/cloud/network/lb/ElasticLoadBalancerManagerImpl.java +++ b/server/src/com/cloud/network/lb/ElasticLoadBalancerManagerImpl.java @@ -34,6 +34,8 @@ import com.cloud.agent.AgentManager; import com.cloud.agent.AgentManager.OnError; import com.cloud.agent.api.Answer; import com.cloud.agent.api.StopAnswer; +import com.cloud.agent.api.check.CheckSshAnswer; +import com.cloud.agent.api.check.CheckSshCommand; import com.cloud.agent.api.routing.LoadBalancerConfigCommand; import com.cloud.agent.api.routing.NetworkElementCommand; import com.cloud.agent.api.to.LoadBalancerTO; @@ -42,6 +44,8 @@ import com.cloud.api.commands.CreateLoadBalancerRuleCmd; import com.cloud.configuration.Config; import com.cloud.configuration.dao.ConfigurationDao; import com.cloud.dc.DataCenter; +import com.cloud.dc.DataCenter.NetworkType; +import com.cloud.dc.DataCenterVO; import com.cloud.dc.Pod; import com.cloud.dc.PodVlanMapVO; import com.cloud.dc.Vlan.VlanType; @@ -61,6 +65,7 @@ import com.cloud.exception.NetworkRuleConflictException; import com.cloud.exception.OperationTimedoutException; import com.cloud.exception.ResourceUnavailableException; import com.cloud.exception.StorageUnavailableException; +import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.network.ElasticLbVmMapVO; import com.cloud.network.IPAddressVO; import com.cloud.network.LoadBalancerVO; @@ -106,9 +111,11 @@ import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.DomainRouterVO; import com.cloud.vm.NicProfile; import com.cloud.vm.ReservationContext; +import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachineGuru; import com.cloud.vm.VirtualMachine.State; +import com.cloud.vm.VirtualMachineGuru; import com.cloud.vm.VirtualMachineManager; import com.cloud.vm.VirtualMachineName; import com.cloud.vm.VirtualMachineProfile; @@ -171,6 +178,8 @@ public class ElasticLoadBalancerManagerImpl implements String _name; String _instance; + static final private String _elbVmNamePrefix = "l"; + static final private String _systemVmType = "elbvm"; boolean _enabled; TrafficType _frontendTrafficType = TrafficType.Guest; @@ -178,6 +187,8 @@ public class ElasticLoadBalancerManagerImpl implements Account _systemAcct; ServiceOfferingVO _elasticLbVmOffering; ScheduledExecutorService _gcThreadPool; + String _mgmtCidr; + String _mgmtHost; int _elasticLbVmRamSize; int _elasticLbvmCpuMHz; @@ -352,6 +363,9 @@ public class ElasticLoadBalancerManagerImpl implements if (_instance == null) { _instance = "VM"; } + _mgmtCidr = _configDao.getValue(Config.ManagementNetwork.key()); + _mgmtHost = _configDao.getValue(Config.ManagementHostIPAdr.key()); + boolean useLocalStorage = Boolean.parseBoolean(configs.get(Config.SystemVMUseLocalStorage.key())); _elasticLbVmRamSize = NumbersUtil.parseInt(configs.get("elastic.lb.vm.ram.size"), DEFAULT_ELB_VM_RAMSIZE); @@ -360,6 +374,8 @@ public class ElasticLoadBalancerManagerImpl implements _elasticLbVmOffering.setUniqueName("Cloud.Com-ElasticLBVm"); _elasticLbVmOffering = _serviceOfferingDao.persistSystemServiceOffering(_elasticLbVmOffering); + + String enabled = _configDao.getValue(Config.ElasticLoadBalancerEnabled.key()); _enabled = (enabled == null) ? false: Boolean.parseBoolean(enabled); if (_enabled) { @@ -371,9 +387,8 @@ public class ElasticLoadBalancerManagerImpl implements } else throw new ConfigurationException("Traffic type for front end of load balancer has to be guest or public; found : " + traffType); _gcThreadPool = Executors.newScheduledThreadPool(1, new NamedThreadFactory("ELBVM-GC")); - _gcThreadPool.scheduleAtFixedRate(new CleanupThread(), 30, 30, TimeUnit.SECONDS); - _itMgr.registerGuru(VirtualMachine.Type.DomainRouter, this); - + _gcThreadPool.scheduleAtFixedRate(new CleanupThread(), gcIntervalMinutes, gcIntervalMinutes, TimeUnit.MINUTES); + _itMgr.registerGuru(VirtualMachine.Type.ElasticLoadBalancerVm, this); } @@ -450,10 +465,11 @@ public class ElasticLoadBalancerManagerImpl implements VMTemplateVO template = _templateDao.findSystemVMTemplate(dcId); - elbVm = new DomainRouterVO(id, _elasticLbVmOffering.getId(), VirtualMachineName.getRouterName(id, _instance), template.getId(), template.getHypervisorType(), template.getGuestOSId(), + elbVm = new DomainRouterVO(id, _elasticLbVmOffering.getId(), VirtualMachineName.getSystemVmName(id, _instance, _elbVmNamePrefix), template.getId(), template.getHypervisorType(), template.getGuestOSId(), owner.getDomainId(), owner.getId(), guestNetwork.getId(), _elasticLbVmOffering.getOfferHA()); elbVm.setRole(Role.LB); elbVm = _itMgr.allocate(elbVm, template, _elasticLbVmOffering, networks, plan, null, owner); + //TODO: create usage stats } State state = elbVm.getState(); @@ -695,66 +711,210 @@ public class ElasticLoadBalancerManagerImpl implements return null; } - @Override - public DomainRouterVO findById(long id) { - // TODO Auto-generated method stub - return null; + public DomainRouterVO findByName(String name) { + if (!VirtualMachineName.isValidSystemVmName(name, _instance, _elbVmNamePrefix)) { + return null; + } + + return _routerDao.findById(VirtualMachineName.getSystemVmId(name)); } @Override - public DomainRouterVO persist(DomainRouterVO vm) { - // TODO Auto-generated method stub - return null; + public DomainRouterVO findById(long id) { + return _routerDao.findById(id); + } + + + @Override + public DomainRouterVO persist(DomainRouterVO elbVm) { + return _routerDao.persist(elbVm); } @Override public boolean finalizeVirtualMachineProfile(VirtualMachineProfile profile, DeployDestination dest, ReservationContext context) { - // TODO Auto-generated method stub - return false; + DomainRouterVO elbVm = profile.getVirtualMachine(); + NetworkVO network = _networkDao.findById(elbVm.getNetworkId()); + + DataCenter dc = dest.getDataCenter(); + + StringBuilder buf = profile.getBootArgsBuilder(); + buf.append(" template=domP type=" + _systemVmType); + buf.append(" name=").append(profile.getHostName()); + NicProfile controlNic = null; + String defaultDns1 = null; + String defaultDns2 = null; + + for (NicProfile nic : profile.getNics()) { + int deviceId = nic.getDeviceId(); + buf.append(" eth").append(deviceId).append("ip=").append(nic.getIp4Address()); + buf.append(" eth").append(deviceId).append("mask=").append(nic.getNetmask()); + if (nic.isDefaultNic()) { + buf.append(" gateway=").append(nic.getGateway()); + defaultDns1 = nic.getDns1(); + defaultDns2 = nic.getDns2(); + } + if (nic.getTrafficType() == TrafficType.Management) { + buf.append(" localgw=").append(dest.getPod().getGateway()); + } else if (nic.getTrafficType() == TrafficType.Control) { + // control command is sent over management network in VMware + if (dest.getHost().getHypervisorType() == HypervisorType.VMware) { + if (s_logger.isInfoEnabled()) { + s_logger.info("Check if we need to add management server explicit route to elb vm. pod cidr: " + dest.getPod().getCidrAddress() + "/" + dest.getPod().getCidrSize() + + ", pod gateway: " + dest.getPod().getGateway() + ", management host: " + _mgmtHost); + } + + if (s_logger.isDebugEnabled()) { + s_logger.debug("Added management server explicit route to elb vm."); + } + // always add management explicit route, for basic networking setup + buf.append(" mgmtcidr=").append(_mgmtCidr); + buf.append(" localgw=").append(dest.getPod().getGateway()); + + if (dc.getNetworkType() == NetworkType.Basic) { + // ask elb vm to setup SSH on guest network + buf.append(" sshonguest=true"); + } + } + + controlNic = nic; + } + } + String domain = network.getNetworkDomain(); + if (domain != null) { + buf.append(" domain=" + domain); + } + + buf.append(" dns1=").append(defaultDns1); + if (defaultDns2 != null) { + buf.append(" dns2=").append(defaultDns2); + } + + if (s_logger.isDebugEnabled()) { + s_logger.debug("Boot Args for " + profile + ": " + buf.toString()); + } + + if (controlNic == null) { + throw new CloudRuntimeException("Didn't start a control port"); + } + + return true; } @Override public boolean finalizeDeployment(Commands cmds, VirtualMachineProfile profile, DeployDestination dest, ReservationContext context) throws ResourceUnavailableException { - // TODO Auto-generated method stub - return false; + DomainRouterVO elbVm = profile.getVirtualMachine(); + + List nics = profile.getNics(); + for (NicProfile nic : nics) { + if (nic.getTrafficType() == TrafficType.Public) { + elbVm.setPublicIpAddress(nic.getIp4Address()); + elbVm.setPublicNetmask(nic.getNetmask()); + elbVm.setPublicMacAddress(nic.getMacAddress()); + } else if (nic.getTrafficType() == TrafficType.Guest) { + elbVm.setGuestIpAddress(nic.getIp4Address()); + } else if (nic.getTrafficType() == TrafficType.Control) { + elbVm.setPrivateIpAddress(nic.getIp4Address()); + elbVm.setPrivateMacAddress(nic.getMacAddress()); + } + } + _routerDao.update(elbVm.getId(), elbVm); + + finalizeCommandsOnStart(cmds, profile); + return true; } @Override public boolean finalizeStart(VirtualMachineProfile profile, long hostId, Commands cmds, ReservationContext context) { - // TODO Auto-generated method stub - return false; + CheckSshAnswer answer = (CheckSshAnswer) cmds.getAnswer("checkSsh"); + if (answer == null || !answer.getResult()) { + s_logger.warn("Unable to ssh to the ELB VM: " + answer.getDetails()); + return false; + } + + return true; } @Override public boolean finalizeCommandsOnStart(Commands cmds, VirtualMachineProfile profile) { - // TODO Auto-generated method stub - return false; + DomainRouterVO elbVm = profile.getVirtualMachine(); + DataCenterVO dcVo = _dcDao.findById(elbVm.getDataCenterIdToDeployIn()); + + NicProfile controlNic = null; + + if(profile.getHypervisorType() == HypervisorType.VMware && dcVo.getNetworkType() == NetworkType.Basic) { + // TODO this is a ugly to test hypervisor type here + // for basic network mode, we will use the guest NIC for control NIC + for (NicProfile nic : profile.getNics()) { + if (nic.getTrafficType() == TrafficType.Guest && nic.getIp4Address() != null) { + controlNic = nic; + } + } + } else { + for (NicProfile nic : profile.getNics()) { + if (nic.getTrafficType() == TrafficType.Control && nic.getIp4Address() != null) { + controlNic = nic; + } + } + } + + if (controlNic == null) { + s_logger.error("Control network doesn't exist for the ELB vm " + elbVm); + return false; + } + + cmds.addCommand("checkSsh", new CheckSshCommand(profile.getInstanceName(), controlNic.getIp4Address(), 3922, 5, 20)); + + // Re-apply load balancing rules + List lbs = _elbVmMapDao.listLbsForElbVm(elbVm.getId()); + List lbRules = new ArrayList(); + for (LoadBalancerVO lb : lbs) { + List dstList = _lbMgr.getExistingDestinations(lb.getId()); + LoadBalancingRule loadBalancing = new LoadBalancingRule(lb, dstList); + lbRules.add(loadBalancing); + } + + s_logger.debug("Found " + lbRules.size() + " load balancing rule(s) to apply as a part of ELB vm " + elbVm + " start."); + if (!lbRules.isEmpty()) { + createApplyLoadBalancingRulesCommands(lbRules, elbVm, cmds); + } + + return true; } @Override public void finalizeStop(VirtualMachineProfile profile, StopAnswer answer) { - // TODO Auto-generated method stub - + if (answer != null) { + VMInstanceVO vm = profile.getVirtualMachine(); + DomainRouterVO elbVm = _routerDao.findById(vm.getId()); + processStopOrRebootAnswer(elbVm, answer); + } + } + + public void processStopOrRebootAnswer(final DomainRouterVO elbVm, Answer answer) { + //TODO: process network usage stats } @Override public void finalizeExpunge(DomainRouterVO vm) { - // TODO Auto-generated method stub + // no-op } @Override public Long convertToId(String vmName) { - // TODO Auto-generated method stub - return null; + if (!VirtualMachineName.isValidSystemVmName(vmName, _instance, _elbVmNamePrefix)) { + return null; + } + + return VirtualMachineName.getSystemVmId(vmName); } } diff --git a/server/src/com/cloud/network/lb/dao/ElasticLbVmMapDao.java b/server/src/com/cloud/network/lb/dao/ElasticLbVmMapDao.java index 9a8921cb0f7..ff529012923 100644 --- a/server/src/com/cloud/network/lb/dao/ElasticLbVmMapDao.java +++ b/server/src/com/cloud/network/lb/dao/ElasticLbVmMapDao.java @@ -21,6 +21,7 @@ package com.cloud.network.lb.dao; import java.util.List; import com.cloud.network.ElasticLbVmMapVO; +import com.cloud.network.LoadBalancerVO; import com.cloud.utils.db.GenericDao; import com.cloud.vm.DomainRouterVO; @@ -33,5 +34,6 @@ public interface ElasticLbVmMapDao extends GenericDao { List listByLbId(long lbId); int deleteLB(long lbId); List listUnusedElbVms(); + List listLbsForElbVm(long elbVmId); } diff --git a/server/src/com/cloud/network/lb/dao/ElasticLbVmMapDaoImpl.java b/server/src/com/cloud/network/lb/dao/ElasticLbVmMapDaoImpl.java index bd06aa7e017..590567a27da 100644 --- a/server/src/com/cloud/network/lb/dao/ElasticLbVmMapDaoImpl.java +++ b/server/src/com/cloud/network/lb/dao/ElasticLbVmMapDaoImpl.java @@ -16,13 +16,17 @@ * */ -package com.cloud.network.lb.dao; - +package com.cloud.network.lb.dao; + import java.util.List; import javax.ejb.Local; import com.cloud.network.ElasticLbVmMapVO; +import com.cloud.network.LoadBalancerVO; +import com.cloud.network.dao.LoadBalancerDao; +import com.cloud.network.dao.LoadBalancerDaoImpl; +import com.cloud.network.router.VirtualRouter.Role; import com.cloud.utils.component.ComponentLocator; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.JoinBuilder.JoinType; @@ -31,21 +35,27 @@ import com.cloud.utils.db.SearchCriteria; import com.cloud.vm.DomainRouterVO; import com.cloud.vm.dao.DomainRouterDao; import com.cloud.vm.dao.DomainRouterDaoImpl; - -@Local(value={ElasticLbVmMapDao.class}) + +@Local(value={ElasticLbVmMapDao.class}) public class ElasticLbVmMapDaoImpl extends GenericDaoBase implements ElasticLbVmMapDao { protected final DomainRouterDao _routerDao = ComponentLocator.inject(DomainRouterDaoImpl.class); + protected final LoadBalancerDao _loadbalancerDao = ComponentLocator.inject(LoadBalancerDaoImpl.class); + protected final SearchBuilder AllFieldsSearch; protected final SearchBuilder UnusedVmSearch; + protected final SearchBuilder LoadBalancersForElbVmSearch; - protected final SearchBuilder ElbVmSearch; - - protected ElasticLbVmMapDaoImpl() { + + protected final SearchBuilder ElbVmSearch; + + protected final SearchBuilder LoadBalancerSearch; + + protected ElasticLbVmMapDaoImpl() { AllFieldsSearch = createSearchBuilder(); - AllFieldsSearch.and("ipId", AllFieldsSearch.entity().getIpAddressId(), SearchCriteria.Op.EQ); - AllFieldsSearch.and("lbId", AllFieldsSearch.entity().getLbId(), SearchCriteria.Op.EQ); - AllFieldsSearch.and("elbVmId", AllFieldsSearch.entity().getElbVmId(), SearchCriteria.Op.EQ); + AllFieldsSearch.and("ipId", AllFieldsSearch.entity().getIpAddressId(), SearchCriteria.Op.EQ); + AllFieldsSearch.and("lbId", AllFieldsSearch.entity().getLbId(), SearchCriteria.Op.EQ); + AllFieldsSearch.and("elbVmId", AllFieldsSearch.entity().getElbVmId(), SearchCriteria.Op.EQ); AllFieldsSearch.done(); ElbVmSearch = _routerDao.createSearchBuilder(); @@ -55,36 +65,43 @@ public class ElasticLbVmMapDaoImpl extends GenericDaoBase sc = AllFieldsSearch.create(); + sc.setParameters("lbId", lbId); + sc.setParameters("elbVmId", elbVmId); + return findOneBy(sc); + } + + @Override + public List listByLbId(long lbId) { + SearchCriteria sc = AllFieldsSearch.create(); + sc.setParameters("lbId", lbId); + return listBy(sc); + } + + @Override + public List listByElbVmId(long elbVmId) { + SearchCriteria sc = AllFieldsSearch.create(); + sc.setParameters("elbVmId", elbVmId); + return listBy(sc); + } - } - @Override - public ElasticLbVmMapVO findOneByLbIdAndElbVmId(long lbId, long elbVmId) { - SearchCriteria sc = AllFieldsSearch.create(); - sc.setParameters("lbId", lbId); - sc.setParameters("elbVmId", elbVmId); - return findOneBy(sc); - } - - @Override - public List listByLbId(long lbId) { - SearchCriteria sc = AllFieldsSearch.create(); - sc.setParameters("lbId", lbId); - return listBy(sc); - } - - @Override - public List listByElbVmId(long elbVmId) { - SearchCriteria sc = AllFieldsSearch.create(); - sc.setParameters("elbVmId", elbVmId); - return listBy(sc); - } - - @Override public int deleteLB(long lbId) { SearchCriteria sc = AllFieldsSearch.create(); - sc.setParameters("lbId", lbId); - return super.expunge(sc); + sc.setParameters("lbId", lbId); + return super.expunge(sc); } @Override @@ -106,5 +123,12 @@ public class ElasticLbVmMapDaoImpl extends GenericDaoBase sc = ElbVmSearch.create(); return _routerDao.search(sc, null); } - -} + + @Override + public List listLbsForElbVm(long elbVmId) { + SearchCriteria sc = LoadBalancerSearch.create(); + sc.setJoinParameters("LoadBalancersForElbVm", "elbVmId", elbVmId); + return _loadbalancerDao.search(sc, null); + } + +} diff --git a/server/test/com/cloud/network/dao/ElbVmMapDaoTest.java b/server/test/com/cloud/network/dao/ElbVmMapDaoTest.java index ff1b3ec5f60..eb92875fa4c 100644 --- a/server/test/com/cloud/network/dao/ElbVmMapDaoTest.java +++ b/server/test/com/cloud/network/dao/ElbVmMapDaoTest.java @@ -5,6 +5,7 @@ import java.util.List; import junit.framework.TestCase; import com.cloud.network.ElasticLbVmMapVO; +import com.cloud.network.LoadBalancerVO; import com.cloud.network.lb.dao.ElasticLbVmMapDaoImpl; import com.cloud.utils.component.ComponentLocator; import com.cloud.vm.DomainRouterVO; @@ -30,4 +31,15 @@ public class ElbVmMapDaoTest extends TestCase { System.out.println("Found"); } } + + public void testFindLB() { + ElasticLbVmMapDaoImpl dao = ComponentLocator.inject(ElasticLbVmMapDaoImpl.class); + + List lbs = dao.listLbsForElbVm(10); + if (lbs == null) { + System.out.println("Not Found"); + } else { + System.out.println("Found"); + } + } }