From 14dd3f07d731c745bba0d8196b70275e32bb716d Mon Sep 17 00:00:00 2001 From: Alena Prokharchyk Date: Tue, 21 Aug 2012 15:24:44 -0700 Subject: [PATCH] Fixed line endings in VirtualNetworkApplianceManagerImpl --- .../VirtualNetworkApplianceManagerImpl.java | 6718 ++++++++--------- 1 file changed, 3359 insertions(+), 3359 deletions(-) diff --git a/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java b/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java index 082c6302127..2e1ec2a83ed 100755 --- a/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java +++ b/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java @@ -1,1600 +1,1600 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloud.network.router; - -import java.util.ArrayList; -import java.util.Calendar; -import java.util.Collections; -import java.util.Comparator; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TimeZone; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; - -import javax.ejb.Local; -import javax.naming.ConfigurationException; - -import org.apache.log4j.Logger; - -import com.cloud.agent.AgentManager; -import com.cloud.agent.AgentManager.OnError; -import com.cloud.agent.Listener; -import com.cloud.agent.api.AgentControlAnswer; -import com.cloud.agent.api.AgentControlCommand; -import com.cloud.agent.api.Answer; -import com.cloud.agent.api.BumpUpPriorityCommand; -import com.cloud.agent.api.CheckRouterAnswer; -import com.cloud.agent.api.CheckRouterCommand; -import com.cloud.agent.api.CheckS2SVpnConnectionsAnswer; -import com.cloud.agent.api.CheckS2SVpnConnectionsCommand; -import com.cloud.agent.api.Command; -import com.cloud.agent.api.GetDomRVersionAnswer; -import com.cloud.agent.api.GetDomRVersionCmd; -import com.cloud.agent.api.ModifySshKeysCommand; -import com.cloud.agent.api.NetworkUsageAnswer; -import com.cloud.agent.api.NetworkUsageCommand; -import com.cloud.agent.api.StartupCommand; -import com.cloud.agent.api.StopAnswer; -import com.cloud.agent.api.check.CheckSshAnswer; -import com.cloud.agent.api.check.CheckSshCommand; -import com.cloud.agent.api.routing.DhcpEntryCommand; -import com.cloud.agent.api.routing.IpAssocCommand; -import com.cloud.agent.api.routing.LoadBalancerConfigCommand; -import com.cloud.agent.api.routing.NetworkElementCommand; -import com.cloud.agent.api.routing.RemoteAccessVpnCfgCommand; -import com.cloud.agent.api.routing.SavePasswordCommand; -import com.cloud.agent.api.routing.SetFirewallRulesCommand; -import com.cloud.agent.api.routing.SetPortForwardingRulesCommand; -import com.cloud.agent.api.routing.SetPortForwardingRulesVpcCommand; -import com.cloud.agent.api.routing.SetStaticNatRulesCommand; -import com.cloud.agent.api.routing.VmDataCommand; -import com.cloud.agent.api.routing.VpnUsersCfgCommand; -import com.cloud.agent.api.to.FirewallRuleTO; -import com.cloud.agent.api.to.IpAddressTO; -import com.cloud.agent.api.to.LoadBalancerTO; -import com.cloud.agent.api.to.NicTO; -import com.cloud.agent.api.to.PortForwardingRuleTO; -import com.cloud.agent.api.to.StaticNatRuleTO; -import com.cloud.agent.api.to.VirtualMachineTO; -import com.cloud.agent.manager.Commands; -import com.cloud.alert.AlertManager; -import com.cloud.api.commands.UpgradeRouterCmd; -import com.cloud.cluster.ManagementServerHostVO; -import com.cloud.cluster.ManagementServerNode; -import com.cloud.cluster.dao.ManagementServerHostDao; -import com.cloud.configuration.Config; -import com.cloud.configuration.ConfigurationManager; -import com.cloud.configuration.ZoneConfig; -import com.cloud.configuration.dao.ConfigurationDao; -import com.cloud.dc.ClusterVO; -import com.cloud.dc.DataCenter; -import com.cloud.dc.DataCenter.NetworkType; -import com.cloud.dc.DataCenterVO; -import com.cloud.dc.HostPodVO; -import com.cloud.dc.Pod; -import com.cloud.dc.dao.ClusterDao; -import com.cloud.dc.dao.DataCenterDao; -import com.cloud.dc.dao.HostPodDao; -import com.cloud.dc.dao.VlanDao; -import com.cloud.deploy.DataCenterDeployment; -import com.cloud.deploy.DeployDestination; -import com.cloud.deploy.DeploymentPlan; -import com.cloud.deploy.DeploymentPlanner.ExcludeList; -import com.cloud.event.ActionEvent; -import com.cloud.event.EventTypes; -import com.cloud.exception.AgentUnavailableException; -import com.cloud.exception.ConcurrentOperationException; -import com.cloud.exception.ConnectionException; -import com.cloud.exception.InsufficientAddressCapacityException; -import com.cloud.exception.InsufficientCapacityException; -import com.cloud.exception.InsufficientServerCapacityException; -import com.cloud.exception.InsufficientVirtualNetworkCapcityException; -import com.cloud.exception.InvalidParameterValueException; -import com.cloud.exception.OperationTimedoutException; -import com.cloud.exception.ResourceUnavailableException; -import com.cloud.exception.StorageUnavailableException; -import com.cloud.host.HostVO; -import com.cloud.host.Status; -import com.cloud.host.dao.HostDao; -import com.cloud.hypervisor.Hypervisor.HypervisorType; -import com.cloud.network.IPAddressVO; -import com.cloud.network.IpAddress; -import com.cloud.network.LoadBalancerVO; -import com.cloud.network.Network; -import com.cloud.network.Network.GuestType; -import com.cloud.network.Network.Provider; -import com.cloud.network.Network.Service; -import com.cloud.network.NetworkManager; -import com.cloud.network.NetworkVO; -import com.cloud.network.Networks.BroadcastDomainType; -import com.cloud.network.Networks.IsolationType; -import com.cloud.network.Networks.TrafficType; -import com.cloud.network.PhysicalNetworkServiceProvider; -import com.cloud.network.PublicIpAddress; -import com.cloud.network.RemoteAccessVpn; -import com.cloud.network.Site2SiteCustomerGateway; -import com.cloud.network.Site2SiteVpnConnection; -import com.cloud.network.Site2SiteVpnConnectionVO; -import com.cloud.network.SshKeysDistriMonitor; -import com.cloud.network.VirtualNetworkApplianceService; -import com.cloud.network.VirtualRouterProvider; -import com.cloud.network.VirtualRouterProvider.VirtualRouterProviderType; -import com.cloud.network.VpnUser; -import com.cloud.network.VpnUserVO; -import com.cloud.network.addr.PublicIp; -import com.cloud.network.dao.FirewallRulesDao; -import com.cloud.network.dao.IPAddressDao; -import com.cloud.network.dao.LoadBalancerDao; -import com.cloud.network.dao.LoadBalancerVMMapDao; -import com.cloud.network.dao.NetworkDao; -import com.cloud.network.dao.PhysicalNetworkServiceProviderDao; -import com.cloud.network.dao.RemoteAccessVpnDao; -import com.cloud.network.dao.Site2SiteCustomerGatewayDao; -import com.cloud.network.dao.Site2SiteVpnConnectionDao; -import com.cloud.network.dao.Site2SiteVpnGatewayDao; -import com.cloud.network.dao.VirtualRouterProviderDao; -import com.cloud.network.dao.VpnUserDao; -import com.cloud.network.lb.LoadBalancingRule; -import com.cloud.network.lb.LoadBalancingRule.LbDestination; -import com.cloud.network.lb.LoadBalancingRule.LbStickinessPolicy; -import com.cloud.network.lb.LoadBalancingRulesManager; -import com.cloud.network.router.VirtualRouter.RedundantState; -import com.cloud.network.router.VirtualRouter.Role; -import com.cloud.network.rules.FirewallRule; -import com.cloud.network.rules.FirewallRule.Purpose; -import com.cloud.network.rules.PortForwardingRule; -import com.cloud.network.rules.RulesManager; -import com.cloud.network.rules.StaticNat; -import com.cloud.network.rules.StaticNatImpl; -import com.cloud.network.rules.StaticNatRule; -import com.cloud.network.rules.dao.PortForwardingRulesDao; -import com.cloud.network.vpn.Site2SiteVpnManager; -import com.cloud.offering.ServiceOffering; -import com.cloud.offerings.NetworkOfferingVO; -import com.cloud.offerings.dao.NetworkOfferingDao; -import com.cloud.resource.ResourceManager; -import com.cloud.service.ServiceOfferingVO; -import com.cloud.service.dao.ServiceOfferingDao; -import com.cloud.storage.GuestOSVO; -import com.cloud.storage.VMTemplateVO; -import com.cloud.storage.Volume.Type; -import com.cloud.storage.VolumeVO; -import com.cloud.storage.dao.GuestOSDao; -import com.cloud.storage.dao.VMTemplateDao; -import com.cloud.storage.dao.VolumeDao; -import com.cloud.user.Account; -import com.cloud.user.AccountManager; -import com.cloud.user.User; -import com.cloud.user.UserContext; -import com.cloud.user.UserStatisticsVO; -import com.cloud.user.UserStatsLogVO; -import com.cloud.user.UserVO; -import com.cloud.user.dao.UserDao; -import com.cloud.user.dao.UserStatisticsDao; -import com.cloud.user.dao.UserStatsLogDao; -import com.cloud.uservm.UserVm; -import com.cloud.utils.NumbersUtil; -import com.cloud.utils.Pair; -import com.cloud.utils.PasswordGenerator; -import com.cloud.utils.StringUtils; -import com.cloud.utils.component.ComponentLocator; -import com.cloud.utils.component.Inject; -import com.cloud.utils.concurrency.NamedThreadFactory; -import com.cloud.utils.db.DB; -import com.cloud.utils.db.Filter; -import com.cloud.utils.db.GlobalLock; -import com.cloud.utils.db.Transaction; -import com.cloud.utils.exception.CloudRuntimeException; -import com.cloud.utils.net.MacAddress; -import com.cloud.utils.net.NetUtils; -import com.cloud.vm.DomainRouterVO; -import com.cloud.vm.Nic; -import com.cloud.vm.NicProfile; -import com.cloud.vm.NicVO; -import com.cloud.vm.ReservationContext; -import com.cloud.vm.ReservationContextImpl; -import com.cloud.vm.UserVmVO; -import com.cloud.vm.VMInstanceVO; -import com.cloud.vm.VirtualMachine; -import com.cloud.vm.VirtualMachine.State; -import com.cloud.vm.VirtualMachineGuru; -import com.cloud.vm.VirtualMachineManager; -import com.cloud.vm.VirtualMachineName; -import com.cloud.vm.VirtualMachineProfile; -import com.cloud.vm.VirtualMachineProfile.Param; -import com.cloud.vm.dao.DomainRouterDao; -import com.cloud.vm.dao.NicDao; -import com.cloud.vm.dao.UserVmDao; -import com.cloud.vm.dao.UserVmDetailsDao; - -/** - * VirtualNetworkApplianceManagerImpl manages the different types of virtual network appliances available in the Cloud Stack. - */ -@Local(value = { VirtualNetworkApplianceManager.class, VirtualNetworkApplianceService.class }) -public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplianceManager, VirtualNetworkApplianceService, - VirtualMachineGuru, Listener { - private static final Logger s_logger = Logger.getLogger(VirtualNetworkApplianceManagerImpl.class); - - String _name; - @Inject - DataCenterDao _dcDao = null; - @Inject - VlanDao _vlanDao = null; - @Inject - FirewallRulesDao _rulesDao = null; - @Inject - LoadBalancerDao _loadBalancerDao = null; - @Inject - LoadBalancerVMMapDao _loadBalancerVMMapDao = null; - @Inject - IPAddressDao _ipAddressDao = null; - @Inject - VMTemplateDao _templateDao = null; - @Inject - DomainRouterDao _routerDao = null; - @Inject - UserDao _userDao = null; - @Inject - UserStatisticsDao _userStatsDao = null; - @Inject - HostDao _hostDao = null; - @Inject - ConfigurationDao _configDao; - @Inject - HostPodDao _podDao = null; - @Inject - UserStatsLogDao _userStatsLogDao = null; - @Inject - AgentManager _agentMgr; - @Inject - AlertManager _alertMgr; - @Inject - AccountManager _accountMgr; - @Inject - ConfigurationManager _configMgr; - @Inject - ServiceOfferingDao _serviceOfferingDao = null; - @Inject - UserVmDao _userVmDao; - @Inject - UserStatisticsDao _statsDao = null; - @Inject - NetworkOfferingDao _networkOfferingDao = null; - @Inject - GuestOSDao _guestOSDao = null; - @Inject - NetworkManager _networkMgr; - @Inject - VirtualMachineManager _itMgr; - @Inject - VpnUserDao _vpnUsersDao; - @Inject - RemoteAccessVpnDao _remoteAccessVpnDao; - @Inject - RulesManager _rulesMgr; - @Inject - NetworkDao _networkDao; - @Inject - LoadBalancingRulesManager _lbMgr; - @Inject - PortForwardingRulesDao _pfRulesDao; - @Inject - RemoteAccessVpnDao _vpnDao; - @Inject - NicDao _nicDao; - @Inject - VolumeDao _volumeDao = null; - @Inject - UserVmDetailsDao _vmDetailsDao; - @Inject - ClusterDao _clusterDao; - @Inject - ResourceManager _resourceMgr; - @Inject - PhysicalNetworkServiceProviderDao _physicalProviderDao; - @Inject - VirtualRouterProviderDao _vrProviderDao; - @Inject - ManagementServerHostDao _msHostDao; - @Inject - Site2SiteCustomerGatewayDao _s2sCustomerGatewayDao; - @Inject - Site2SiteVpnGatewayDao _s2sVpnGatewayDao; - @Inject - Site2SiteVpnConnectionDao _s2sVpnConnectionDao; - @Inject - Site2SiteVpnManager _s2sVpnMgr; - - int _routerRamSize; - int _routerCpuMHz; - int _retry = 2; - String _instance; - String _mgmt_host; - String _mgmt_cidr; - - int _routerStatsInterval = 300; - int _routerCheckInterval = 30; - protected ServiceOfferingVO _offering; - private String _dnsBasicZoneUpdates = "all"; - - private boolean _disable_rp_filter = false; - int _routerExtraPublicNics = 2; - private int _usageAggregationRange = 1440; - private String _usageTimeZone = "GMT"; - private final long mgmtSrvrId = MacAddress.getMacAddress().toLong(); - private static final int ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_COOPERATION = 5; // 5 seconds - - ScheduledExecutorService _executor; - ScheduledExecutorService _checkExecutor; - ScheduledExecutorService _networkStatsUpdateExecutor; - - Account _systemAcct; - - @Override - public boolean sendSshKeysToHost(Long hostId, String pubKey, String prvKey) { - ModifySshKeysCommand cmd = new ModifySshKeysCommand(pubKey, prvKey); - final Answer answer = _agentMgr.easySend(hostId, cmd); - - if (answer != null) { - return true; - } else { - return false; - } - } - - @Override - public VirtualRouter destroyRouter(final long routerId) throws ResourceUnavailableException, ConcurrentOperationException { - UserContext context = UserContext.current(); - User user = _accountMgr.getActiveUser(context.getCallerUserId()); - - if (s_logger.isDebugEnabled()) { - s_logger.debug("Attempting to destroy router " + routerId); - } - - DomainRouterVO router = _routerDao.findById(routerId); - if (router == null) { - return null; - } - - _accountMgr.checkAccess(context.getCaller(), null, true, router); - - boolean result = _itMgr.expunge(router, user, _accountMgr.getAccount(router.getAccountId())); - - if (result) { - return router; - } - return null; - } - - @Override - @DB - public VirtualRouter upgradeRouter(UpgradeRouterCmd cmd) { - Long routerId = cmd.getId(); - Long serviceOfferingId = cmd.getServiceOfferingId(); - Account caller = UserContext.current().getCaller(); - - DomainRouterVO router = _routerDao.findById(routerId); - if (router == null) { - throw new InvalidParameterValueException("Unable to find router with id " + routerId); - } - - _accountMgr.checkAccess(caller, null, true, router); - - if (router.getServiceOfferingId() == serviceOfferingId) { - s_logger.debug("Router: " + routerId + "already has service offering: " + serviceOfferingId); - return _routerDao.findById(routerId); - } - - ServiceOffering newServiceOffering = _configMgr.getServiceOffering(serviceOfferingId); - if (newServiceOffering == null) { - throw new InvalidParameterValueException("Unable to find service offering with id " + serviceOfferingId); - } - - // check if it is a system service offering, if yes return with error as it cannot be used for user vms - if (!newServiceOffering.getSystemUse()) { - throw new InvalidParameterValueException("Cannot upgrade router vm to a non system service offering " + serviceOfferingId); - } - - // Check that the router is stopped - if (!router.getState().equals(State.Stopped)) { - s_logger.warn("Unable to upgrade router " + router.toString() + " in state " + router.getState()); - throw new InvalidParameterValueException("Unable to upgrade router " + router.toString() + " in state " + router.getState() - + "; make sure the router is stopped and not in an error state before upgrading."); - } - - ServiceOfferingVO currentServiceOffering = _serviceOfferingDao.findById(router.getServiceOfferingId()); - - // Check that the service offering being upgraded to has the same storage pool preference as the VM's current service - // offering - if (currentServiceOffering.getUseLocalStorage() != newServiceOffering.getUseLocalStorage()) { - throw new InvalidParameterValueException("Can't upgrade, due to new local storage status : " + - newServiceOffering.getUseLocalStorage() + " is different from " - + "curruent local storage status: " + currentServiceOffering.getUseLocalStorage()); - } - - router.setServiceOfferingId(serviceOfferingId); - if (_routerDao.update(routerId, router)) { - return _routerDao.findById(routerId); - } else { - throw new CloudRuntimeException("Unable to upgrade router " + routerId); - } - - } - - @Override - public boolean savePasswordToRouter(Network network, NicProfile nic, VirtualMachineProfile profile, List routers) throws ResourceUnavailableException { - if (routers == null || routers.isEmpty()) { - s_logger.warn("Unable save password, router doesn't exist in network " + network.getId()); - throw new CloudRuntimeException("Unable to save password to router"); - } - - UserVm userVm = profile.getVirtualMachine(); - String password = (String) profile.getParameter(Param.VmPassword); - String encodedPassword = PasswordGenerator.rot13(password); - DataCenter dc = _dcDao.findById(userVm.getDataCenterIdToDeployIn()); - - boolean result = true; - for (VirtualRouter router : routers) { - boolean sendPassword = true; - if (dc.getNetworkType() == NetworkType.Basic && userVm.getPodIdToDeployIn().longValue() != router.getPodIdToDeployIn().longValue()) { - sendPassword = false; - } - - if (sendPassword) { - Commands cmds = new Commands(OnError.Continue); - SavePasswordCommand cmd = new SavePasswordCommand(encodedPassword, nic.getIp4Address(), userVm.getHostName()); - cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); - cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); - DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); - cmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); - cmds.addCommand("password", cmd); - - result = result && sendCommandsToRouter(router, cmds); - } - } - return result; - } - - @Override @ActionEvent(eventType = EventTypes.EVENT_ROUTER_STOP, eventDescription = "stopping router Vm", async = true) - public VirtualRouter stopRouter(long routerId, boolean forced) throws ResourceUnavailableException, ConcurrentOperationException { - UserContext context = UserContext.current(); - Account account = context.getCaller(); - - // verify parameters - DomainRouterVO router = _routerDao.findById(routerId); - if (router == null) { - throw new InvalidParameterValueException("Unable to find router by id " + routerId + "."); - } - - _accountMgr.checkAccess(account, null, true, router); - - UserVO user = _userDao.findById(UserContext.current().getCallerUserId()); - - VirtualRouter virtualRouter = stop(router, forced, user, account); - if(virtualRouter == null){ - throw new CloudRuntimeException("Failed to stop router with id " + routerId); - } - return virtualRouter; - } - - @DB - public void processStopOrRebootAnswer(final DomainRouterVO router, Answer answer) { - final Transaction txn = Transaction.currentTxn(); - try { - txn.start(); - //FIXME!!! - UserStats command should grab bytesSent/Received for all guest interfaces of the VR - List routerGuestNtwkIds = _routerDao.getRouterNetworks(router.getId()); - for (Long guestNtwkId : routerGuestNtwkIds) { - final UserStatisticsVO userStats = _userStatsDao.lock(router.getAccountId(), router.getDataCenterIdToDeployIn(), - guestNtwkId, null, router.getId(), router.getType().toString()); - if (userStats != null) { - final long currentBytesRcvd = userStats.getCurrentBytesReceived(); - userStats.setCurrentBytesReceived(0); - userStats.setNetBytesReceived(userStats.getNetBytesReceived() + currentBytesRcvd); - - final long currentBytesSent = userStats.getCurrentBytesSent(); - userStats.setCurrentBytesSent(0); - userStats.setNetBytesSent(userStats.getNetBytesSent() + currentBytesSent); - _userStatsDao.update(userStats.getId(), userStats); - s_logger.debug("Successfully updated user statistics as a part of domR " + router + " reboot/stop"); - } else { - s_logger.warn("User stats were not created for account " + router.getAccountId() + " and dc " + router.getDataCenterIdToDeployIn()); - } - } - - txn.commit(); - } catch (final Exception e) { - txn.rollback(); - throw new CloudRuntimeException("Problem updating stats after reboot/stop ", e); - } - } - - @Override @ActionEvent(eventType = EventTypes.EVENT_ROUTER_REBOOT, eventDescription = "rebooting router Vm", async = true) - public VirtualRouter rebootRouter(long routerId, boolean reprogramNetwork) throws ConcurrentOperationException, - ResourceUnavailableException, InsufficientCapacityException { - Account caller = UserContext.current().getCaller(); - - // verify parameters - DomainRouterVO router = _routerDao.findById(routerId); - if (router == null) { - throw new InvalidParameterValueException("Unable to find domain router with id " + routerId + "."); - } - - _accountMgr.checkAccess(caller, null, true, router); - - // Can reboot domain router only in Running state - if (router == null || router.getState() != State.Running) { - s_logger.warn("Unable to reboot, virtual router is not in the right state " + router.getState()); - throw new ResourceUnavailableException("Unable to reboot domR, it is not in right state " + router.getState(), - DataCenter.class, router.getDataCenterIdToDeployIn()); - } - - UserVO user = _userDao.findById(UserContext.current().getCallerUserId()); - s_logger.debug("Stopping and starting router " + router + " as a part of router reboot"); - - if (stop(router, false, user, caller) != null) { - return startRouter(routerId, reprogramNetwork); - } else { - throw new CloudRuntimeException("Failed to reboot router " + router); - } - } - - @Override - public boolean configure(final String name, final Map params) throws ConfigurationException { - _name = name; - - _executor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("RouterMonitor")); - _checkExecutor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("RouterStatusMonitor")); - _networkStatsUpdateExecutor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("NetworkStatsUpdater")); - - final ComponentLocator locator = ComponentLocator.getCurrentLocator(); - - final Map configs = _configDao.getConfiguration("AgentManager", params); - - _mgmt_host = configs.get("host"); - _routerRamSize = NumbersUtil.parseInt(configs.get("router.ram.size"), DEFAULT_ROUTER_VM_RAMSIZE); - _routerCpuMHz = NumbersUtil.parseInt(configs.get("router.cpu.mhz"), DEFAULT_ROUTER_CPU_MHZ); - - _routerExtraPublicNics = NumbersUtil.parseInt(_configDao.getValue(Config.RouterExtraPublicNics.key()), 2); - - String value = configs.get("start.retry"); - _retry = NumbersUtil.parseInt(value, 2); - - value = configs.get("router.stats.interval"); - _routerStatsInterval = NumbersUtil.parseInt(value, 300); - - value = configs.get("router.check.interval"); - _routerCheckInterval = NumbersUtil.parseInt(value, 30); - - _instance = configs.get("instance.name"); - if (_instance == null) { - _instance = "DEFAULT"; - } - - String rpValue = configs.get("network.disable.rpfilter"); - if (rpValue != null && rpValue.equalsIgnoreCase("true")) { - _disable_rp_filter = true; - } - - _dnsBasicZoneUpdates = String.valueOf(_configDao.getValue(Config.DnsBasicZoneUpdates.key())); - - s_logger.info("Router configurations: " + "ramsize=" + _routerRamSize); - - final UserStatisticsDao statsDao = locator.getDao(UserStatisticsDao.class); - if (statsDao == null) { - throw new ConfigurationException("Unable to get " + UserStatisticsDao.class.getName()); - } - - _agentMgr.registerForHostEvents(new SshKeysDistriMonitor(_agentMgr, _hostDao, _configDao), true, false, false); - _itMgr.registerGuru(VirtualMachine.Type.DomainRouter, this); - - boolean useLocalStorage = Boolean.parseBoolean(configs.get(Config.SystemVMUseLocalStorage.key())); - _offering = new ServiceOfferingVO("System Offering For Software Router", 1, _routerRamSize, _routerCpuMHz, null, - null, true, null, useLocalStorage, true, null, true, VirtualMachine.Type.DomainRouter, true); - _offering.setUniqueName(ServiceOffering.routerDefaultOffUniqueName); - _offering = _serviceOfferingDao.persistSystemServiceOffering(_offering); - - // this can sometimes happen, if DB is manually or programmatically manipulated - if(_offering == null) { - String msg = "Data integrity problem : System Offering For Software router VM has been removed?"; - s_logger.error(msg); - throw new ConfigurationException(msg); - } - - _systemAcct = _accountMgr.getSystemAccount(); - - String aggregationRange = configs.get("usage.stats.job.aggregation.range"); - _usageAggregationRange = NumbersUtil.parseInt(aggregationRange, 1440); - _usageTimeZone = configs.get("usage.aggregation.timezone"); - if(_usageTimeZone == null){ - _usageTimeZone = "GMT"; - } - - _agentMgr.registerForHostEvents(this, true, false, false); - - s_logger.info("DomainRouterManager is configured."); - - return true; - } - - @Override - public String getName() { - return _name; - } - - @Override - public boolean start() { - if (_routerStatsInterval > 0){ - _executor.scheduleAtFixedRate(new NetworkUsageTask(), _routerStatsInterval, _routerStatsInterval, TimeUnit.SECONDS); - }else{ - s_logger.debug("router.stats.interval - " + _routerStatsInterval+ " so not scheduling the router stats thread"); - } - - //Schedule Network stats update task - TimeZone usageTimezone = TimeZone.getTimeZone(_usageTimeZone); - Calendar cal = Calendar.getInstance(usageTimezone); - cal.setTime(new Date()); - long endDate = 0; - int HOURLY_TIME = 60; - final int DAILY_TIME = 60 * 24; - if (_usageAggregationRange == DAILY_TIME) { - cal.roll(Calendar.DAY_OF_YEAR, false); - cal.set(Calendar.HOUR_OF_DAY, 0); - cal.set(Calendar.MINUTE, 0); - cal.set(Calendar.SECOND, 0); - cal.set(Calendar.MILLISECOND, 0); - cal.roll(Calendar.DAY_OF_YEAR, true); - cal.add(Calendar.MILLISECOND, -1); - endDate = cal.getTime().getTime(); - } else if (_usageAggregationRange == HOURLY_TIME) { - cal.roll(Calendar.HOUR_OF_DAY, false); - cal.set(Calendar.MINUTE, 0); - cal.set(Calendar.SECOND, 0); - cal.set(Calendar.MILLISECOND, 0); - cal.roll(Calendar.HOUR_OF_DAY, true); - cal.add(Calendar.MILLISECOND, -1); - endDate = cal.getTime().getTime(); - } else { - endDate = cal.getTime().getTime(); - } - - _networkStatsUpdateExecutor.scheduleAtFixedRate(new NetworkStatsUpdateTask(), (endDate - System.currentTimeMillis()), - (_usageAggregationRange * 60 * 1000), TimeUnit.MILLISECONDS); - - if (_routerCheckInterval > 0) { - _checkExecutor.scheduleAtFixedRate(new CheckRouterTask(), _routerCheckInterval, _routerCheckInterval, TimeUnit.SECONDS); - } else { - s_logger.debug("router.check.interval - " + _routerCheckInterval+ " so not scheduling the redundant router checking thread"); - } - - return true; - } - - @Override - public boolean stop() { - return true; - } - - protected VirtualNetworkApplianceManagerImpl() { - } - - @Override - public Long convertToId(final String vmName) { - if (!VirtualMachineName.isValidRouterName(vmName, _instance)) { - return null; - } - - return VirtualMachineName.getRouterId(vmName); - } - - private VmDataCommand generateVmDataCommand(VirtualRouter router, String vmPrivateIpAddress, String userData, - String serviceOffering, String zoneName, String guestIpAddress, String vmName, - String vmInstanceName, long vmId, String publicKey, long guestNetworkId) { - VmDataCommand cmd = new VmDataCommand(vmPrivateIpAddress, vmName); - - cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); - cmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(guestNetworkId, router.getId())); - cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); - - DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); - cmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); - - cmd.addVmData("userdata", "user-data", userData); - cmd.addVmData("metadata", "service-offering", StringUtils.unicodeEscape(serviceOffering)); - cmd.addVmData("metadata", "availability-zone", StringUtils.unicodeEscape(zoneName)); - cmd.addVmData("metadata", "local-ipv4", guestIpAddress); - cmd.addVmData("metadata", "local-hostname", StringUtils.unicodeEscape(vmName)); - if (dcVo.getNetworkType() == NetworkType.Basic) { - cmd.addVmData("metadata", "public-ipv4", guestIpAddress); - cmd.addVmData("metadata", "public-hostname", StringUtils.unicodeEscape(vmName)); - } else - { - if (router.getPublicIpAddress() == null) { - cmd.addVmData("metadata", "public-ipv4", guestIpAddress); - } else { - cmd.addVmData("metadata", "public-ipv4", router.getPublicIpAddress()); - } - cmd.addVmData("metadata", "public-hostname", router.getPublicIpAddress()); - } - cmd.addVmData("metadata", "instance-id", vmInstanceName); - cmd.addVmData("metadata", "vm-id", String.valueOf(vmId)); - cmd.addVmData("metadata", "public-keys", publicKey); - - String cloudIdentifier = _configDao.getValue("cloud.identifier"); - if (cloudIdentifier == null) { - cloudIdentifier = ""; - } else { - cloudIdentifier = "CloudStack-{" + cloudIdentifier + "}"; - } - cmd.addVmData("metadata", "cloud-identifier", cloudIdentifier); - - return cmd; - } - - protected class NetworkUsageTask implements Runnable { - - public NetworkUsageTask() { - } - - @Override - public void run() { - try{ - final List routers = _routerDao.listByStateAndNetworkType(State.Running, GuestType.Isolated, mgmtSrvrId); - s_logger.debug("Found " + routers.size() + " running routers. "); - - for (DomainRouterVO router : routers) { - String privateIP = router.getPrivateIpAddress(); - - if (privateIP != null) { - List routerNics = _nicDao.listByVmId(router.getId()); - for (Nic routerNic : routerNics) { - Network network = _networkMgr.getNetwork(routerNic.getNetworkId()); - if (network.getTrafficType() == TrafficType.Public) { - boolean forVpc = router.getVpcId() != null; - final NetworkUsageCommand usageCmd = new NetworkUsageCommand(privateIP, router.getHostName(), - forVpc, routerNic.getIp4Address()); - UserStatisticsVO previousStats = _statsDao.findBy(router.getAccountId(), - router.getDataCenterIdToDeployIn(), network.getId(), null, router.getId(), router.getType().toString()); - NetworkUsageAnswer answer = null; - try { - answer = (NetworkUsageAnswer) _agentMgr.easySend(router.getHostId(), usageCmd); - } catch (Exception e) { - s_logger.warn("Error while collecting network stats from router: "+router.getInstanceName()+" from host: "+router.getHostId(), e); - continue; - } - - if (answer != null) { - if (!answer.getResult()) { - s_logger.warn("Error while collecting network stats from router: "+router.getInstanceName()+" from host: "+router.getHostId() + "; details: " + answer.getDetails()); - continue; - } - Transaction txn = Transaction.open(Transaction.CLOUD_DB); - try { - if ((answer.getBytesReceived() == 0) && (answer.getBytesSent() == 0)) { - s_logger.debug("Recieved and Sent bytes are both 0. Not updating user_statistics"); - continue; - } - txn.start(); - UserStatisticsVO stats = _statsDao.lock(router.getAccountId(), - router.getDataCenterIdToDeployIn(), network.getId(), routerNic.getIp4Address(), router.getId(), router.getType().toString()); - if (stats == null) { - s_logger.warn("unable to find stats for account: " + router.getAccountId()); - continue; - } - - if(previousStats != null - && ((previousStats.getCurrentBytesReceived() != stats.getCurrentBytesReceived()) - || (previousStats.getCurrentBytesSent() != stats.getCurrentBytesSent()))){ - s_logger.debug("Router stats changed from the time NetworkUsageCommand was sent. " + - "Ignoring current answer. Router: "+answer.getRouterName()+" Rcvd: " + - answer.getBytesReceived()+ "Sent: " +answer.getBytesSent()); - continue; - } - - if (stats.getCurrentBytesReceived() > answer.getBytesReceived()) { - if (s_logger.isDebugEnabled()) { - s_logger.debug("Received # of bytes that's less than the last one. " + - "Assuming something went wrong and persisting it. Router: " + - answer.getRouterName()+" Reported: " + answer.getBytesReceived() - + " Stored: " + stats.getCurrentBytesReceived()); - } - stats.setNetBytesReceived(stats.getNetBytesReceived() + stats.getCurrentBytesReceived()); - } - stats.setCurrentBytesReceived(answer.getBytesReceived()); - if (stats.getCurrentBytesSent() > answer.getBytesSent()) { - if (s_logger.isDebugEnabled()) { - s_logger.debug("Received # of bytes that's less than the last one. " + - "Assuming something went wrong and persisting it. Router: " + - answer.getRouterName()+" Reported: " + answer.getBytesSent() - + " Stored: " + stats.getCurrentBytesSent()); - } - stats.setNetBytesSent(stats.getNetBytesSent() + stats.getCurrentBytesSent()); - } - stats.setCurrentBytesSent(answer.getBytesSent()); - _statsDao.update(stats.getId(), stats); - txn.commit(); - } catch (Exception e) { - txn.rollback(); - s_logger.warn("Unable to update user statistics for account: " + router.getAccountId() - + " Rx: " + answer.getBytesReceived() + "; Tx: " + answer.getBytesSent()); - } finally { - txn.close(); - } - } - } - } - } - } - } catch (Exception e) { - s_logger.warn("Error while collecting network stats", e); +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.network.router; + +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Collections; +import java.util.Comparator; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TimeZone; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import javax.ejb.Local; +import javax.naming.ConfigurationException; + +import org.apache.log4j.Logger; + +import com.cloud.agent.AgentManager; +import com.cloud.agent.AgentManager.OnError; +import com.cloud.agent.Listener; +import com.cloud.agent.api.AgentControlAnswer; +import com.cloud.agent.api.AgentControlCommand; +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.BumpUpPriorityCommand; +import com.cloud.agent.api.CheckRouterAnswer; +import com.cloud.agent.api.CheckRouterCommand; +import com.cloud.agent.api.CheckS2SVpnConnectionsAnswer; +import com.cloud.agent.api.CheckS2SVpnConnectionsCommand; +import com.cloud.agent.api.Command; +import com.cloud.agent.api.GetDomRVersionAnswer; +import com.cloud.agent.api.GetDomRVersionCmd; +import com.cloud.agent.api.ModifySshKeysCommand; +import com.cloud.agent.api.NetworkUsageAnswer; +import com.cloud.agent.api.NetworkUsageCommand; +import com.cloud.agent.api.StartupCommand; +import com.cloud.agent.api.StopAnswer; +import com.cloud.agent.api.check.CheckSshAnswer; +import com.cloud.agent.api.check.CheckSshCommand; +import com.cloud.agent.api.routing.DhcpEntryCommand; +import com.cloud.agent.api.routing.IpAssocCommand; +import com.cloud.agent.api.routing.LoadBalancerConfigCommand; +import com.cloud.agent.api.routing.NetworkElementCommand; +import com.cloud.agent.api.routing.RemoteAccessVpnCfgCommand; +import com.cloud.agent.api.routing.SavePasswordCommand; +import com.cloud.agent.api.routing.SetFirewallRulesCommand; +import com.cloud.agent.api.routing.SetPortForwardingRulesCommand; +import com.cloud.agent.api.routing.SetPortForwardingRulesVpcCommand; +import com.cloud.agent.api.routing.SetStaticNatRulesCommand; +import com.cloud.agent.api.routing.VmDataCommand; +import com.cloud.agent.api.routing.VpnUsersCfgCommand; +import com.cloud.agent.api.to.FirewallRuleTO; +import com.cloud.agent.api.to.IpAddressTO; +import com.cloud.agent.api.to.LoadBalancerTO; +import com.cloud.agent.api.to.NicTO; +import com.cloud.agent.api.to.PortForwardingRuleTO; +import com.cloud.agent.api.to.StaticNatRuleTO; +import com.cloud.agent.api.to.VirtualMachineTO; +import com.cloud.agent.manager.Commands; +import com.cloud.alert.AlertManager; +import com.cloud.api.commands.UpgradeRouterCmd; +import com.cloud.cluster.ManagementServerHostVO; +import com.cloud.cluster.ManagementServerNode; +import com.cloud.cluster.dao.ManagementServerHostDao; +import com.cloud.configuration.Config; +import com.cloud.configuration.ConfigurationManager; +import com.cloud.configuration.ZoneConfig; +import com.cloud.configuration.dao.ConfigurationDao; +import com.cloud.dc.ClusterVO; +import com.cloud.dc.DataCenter; +import com.cloud.dc.DataCenter.NetworkType; +import com.cloud.dc.DataCenterVO; +import com.cloud.dc.HostPodVO; +import com.cloud.dc.Pod; +import com.cloud.dc.dao.ClusterDao; +import com.cloud.dc.dao.DataCenterDao; +import com.cloud.dc.dao.HostPodDao; +import com.cloud.dc.dao.VlanDao; +import com.cloud.deploy.DataCenterDeployment; +import com.cloud.deploy.DeployDestination; +import com.cloud.deploy.DeploymentPlan; +import com.cloud.deploy.DeploymentPlanner.ExcludeList; +import com.cloud.event.ActionEvent; +import com.cloud.event.EventTypes; +import com.cloud.exception.AgentUnavailableException; +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.ConnectionException; +import com.cloud.exception.InsufficientAddressCapacityException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.InsufficientServerCapacityException; +import com.cloud.exception.InsufficientVirtualNetworkCapcityException; +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.exception.OperationTimedoutException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.exception.StorageUnavailableException; +import com.cloud.host.HostVO; +import com.cloud.host.Status; +import com.cloud.host.dao.HostDao; +import com.cloud.hypervisor.Hypervisor.HypervisorType; +import com.cloud.network.IPAddressVO; +import com.cloud.network.IpAddress; +import com.cloud.network.LoadBalancerVO; +import com.cloud.network.Network; +import com.cloud.network.Network.GuestType; +import com.cloud.network.Network.Provider; +import com.cloud.network.Network.Service; +import com.cloud.network.NetworkManager; +import com.cloud.network.NetworkVO; +import com.cloud.network.Networks.BroadcastDomainType; +import com.cloud.network.Networks.IsolationType; +import com.cloud.network.Networks.TrafficType; +import com.cloud.network.PhysicalNetworkServiceProvider; +import com.cloud.network.PublicIpAddress; +import com.cloud.network.RemoteAccessVpn; +import com.cloud.network.Site2SiteCustomerGateway; +import com.cloud.network.Site2SiteVpnConnection; +import com.cloud.network.Site2SiteVpnConnectionVO; +import com.cloud.network.SshKeysDistriMonitor; +import com.cloud.network.VirtualNetworkApplianceService; +import com.cloud.network.VirtualRouterProvider; +import com.cloud.network.VirtualRouterProvider.VirtualRouterProviderType; +import com.cloud.network.VpnUser; +import com.cloud.network.VpnUserVO; +import com.cloud.network.addr.PublicIp; +import com.cloud.network.dao.FirewallRulesDao; +import com.cloud.network.dao.IPAddressDao; +import com.cloud.network.dao.LoadBalancerDao; +import com.cloud.network.dao.LoadBalancerVMMapDao; +import com.cloud.network.dao.NetworkDao; +import com.cloud.network.dao.PhysicalNetworkServiceProviderDao; +import com.cloud.network.dao.RemoteAccessVpnDao; +import com.cloud.network.dao.Site2SiteCustomerGatewayDao; +import com.cloud.network.dao.Site2SiteVpnConnectionDao; +import com.cloud.network.dao.Site2SiteVpnGatewayDao; +import com.cloud.network.dao.VirtualRouterProviderDao; +import com.cloud.network.dao.VpnUserDao; +import com.cloud.network.lb.LoadBalancingRule; +import com.cloud.network.lb.LoadBalancingRule.LbDestination; +import com.cloud.network.lb.LoadBalancingRule.LbStickinessPolicy; +import com.cloud.network.lb.LoadBalancingRulesManager; +import com.cloud.network.router.VirtualRouter.RedundantState; +import com.cloud.network.router.VirtualRouter.Role; +import com.cloud.network.rules.FirewallRule; +import com.cloud.network.rules.FirewallRule.Purpose; +import com.cloud.network.rules.PortForwardingRule; +import com.cloud.network.rules.RulesManager; +import com.cloud.network.rules.StaticNat; +import com.cloud.network.rules.StaticNatImpl; +import com.cloud.network.rules.StaticNatRule; +import com.cloud.network.rules.dao.PortForwardingRulesDao; +import com.cloud.network.vpn.Site2SiteVpnManager; +import com.cloud.offering.ServiceOffering; +import com.cloud.offerings.NetworkOfferingVO; +import com.cloud.offerings.dao.NetworkOfferingDao; +import com.cloud.resource.ResourceManager; +import com.cloud.service.ServiceOfferingVO; +import com.cloud.service.dao.ServiceOfferingDao; +import com.cloud.storage.GuestOSVO; +import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.Volume.Type; +import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.GuestOSDao; +import com.cloud.storage.dao.VMTemplateDao; +import com.cloud.storage.dao.VolumeDao; +import com.cloud.user.Account; +import com.cloud.user.AccountManager; +import com.cloud.user.User; +import com.cloud.user.UserContext; +import com.cloud.user.UserStatisticsVO; +import com.cloud.user.UserStatsLogVO; +import com.cloud.user.UserVO; +import com.cloud.user.dao.UserDao; +import com.cloud.user.dao.UserStatisticsDao; +import com.cloud.user.dao.UserStatsLogDao; +import com.cloud.uservm.UserVm; +import com.cloud.utils.NumbersUtil; +import com.cloud.utils.Pair; +import com.cloud.utils.PasswordGenerator; +import com.cloud.utils.StringUtils; +import com.cloud.utils.component.ComponentLocator; +import com.cloud.utils.component.Inject; +import com.cloud.utils.concurrency.NamedThreadFactory; +import com.cloud.utils.db.DB; +import com.cloud.utils.db.Filter; +import com.cloud.utils.db.GlobalLock; +import com.cloud.utils.db.Transaction; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.net.MacAddress; +import com.cloud.utils.net.NetUtils; +import com.cloud.vm.DomainRouterVO; +import com.cloud.vm.Nic; +import com.cloud.vm.NicProfile; +import com.cloud.vm.NicVO; +import com.cloud.vm.ReservationContext; +import com.cloud.vm.ReservationContextImpl; +import com.cloud.vm.UserVmVO; +import com.cloud.vm.VMInstanceVO; +import com.cloud.vm.VirtualMachine; +import com.cloud.vm.VirtualMachine.State; +import com.cloud.vm.VirtualMachineGuru; +import com.cloud.vm.VirtualMachineManager; +import com.cloud.vm.VirtualMachineName; +import com.cloud.vm.VirtualMachineProfile; +import com.cloud.vm.VirtualMachineProfile.Param; +import com.cloud.vm.dao.DomainRouterDao; +import com.cloud.vm.dao.NicDao; +import com.cloud.vm.dao.UserVmDao; +import com.cloud.vm.dao.UserVmDetailsDao; + +/** + * VirtualNetworkApplianceManagerImpl manages the different types of virtual network appliances available in the Cloud Stack. + */ +@Local(value = { VirtualNetworkApplianceManager.class, VirtualNetworkApplianceService.class }) +public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplianceManager, VirtualNetworkApplianceService, + VirtualMachineGuru, Listener { + private static final Logger s_logger = Logger.getLogger(VirtualNetworkApplianceManagerImpl.class); + + String _name; + @Inject + DataCenterDao _dcDao = null; + @Inject + VlanDao _vlanDao = null; + @Inject + FirewallRulesDao _rulesDao = null; + @Inject + LoadBalancerDao _loadBalancerDao = null; + @Inject + LoadBalancerVMMapDao _loadBalancerVMMapDao = null; + @Inject + IPAddressDao _ipAddressDao = null; + @Inject + VMTemplateDao _templateDao = null; + @Inject + DomainRouterDao _routerDao = null; + @Inject + UserDao _userDao = null; + @Inject + UserStatisticsDao _userStatsDao = null; + @Inject + HostDao _hostDao = null; + @Inject + ConfigurationDao _configDao; + @Inject + HostPodDao _podDao = null; + @Inject + UserStatsLogDao _userStatsLogDao = null; + @Inject + AgentManager _agentMgr; + @Inject + AlertManager _alertMgr; + @Inject + AccountManager _accountMgr; + @Inject + ConfigurationManager _configMgr; + @Inject + ServiceOfferingDao _serviceOfferingDao = null; + @Inject + UserVmDao _userVmDao; + @Inject + UserStatisticsDao _statsDao = null; + @Inject + NetworkOfferingDao _networkOfferingDao = null; + @Inject + GuestOSDao _guestOSDao = null; + @Inject + NetworkManager _networkMgr; + @Inject + VirtualMachineManager _itMgr; + @Inject + VpnUserDao _vpnUsersDao; + @Inject + RemoteAccessVpnDao _remoteAccessVpnDao; + @Inject + RulesManager _rulesMgr; + @Inject + NetworkDao _networkDao; + @Inject + LoadBalancingRulesManager _lbMgr; + @Inject + PortForwardingRulesDao _pfRulesDao; + @Inject + RemoteAccessVpnDao _vpnDao; + @Inject + NicDao _nicDao; + @Inject + VolumeDao _volumeDao = null; + @Inject + UserVmDetailsDao _vmDetailsDao; + @Inject + ClusterDao _clusterDao; + @Inject + ResourceManager _resourceMgr; + @Inject + PhysicalNetworkServiceProviderDao _physicalProviderDao; + @Inject + VirtualRouterProviderDao _vrProviderDao; + @Inject + ManagementServerHostDao _msHostDao; + @Inject + Site2SiteCustomerGatewayDao _s2sCustomerGatewayDao; + @Inject + Site2SiteVpnGatewayDao _s2sVpnGatewayDao; + @Inject + Site2SiteVpnConnectionDao _s2sVpnConnectionDao; + @Inject + Site2SiteVpnManager _s2sVpnMgr; + + int _routerRamSize; + int _routerCpuMHz; + int _retry = 2; + String _instance; + String _mgmt_host; + String _mgmt_cidr; + + int _routerStatsInterval = 300; + int _routerCheckInterval = 30; + protected ServiceOfferingVO _offering; + private String _dnsBasicZoneUpdates = "all"; + + private boolean _disable_rp_filter = false; + int _routerExtraPublicNics = 2; + private int _usageAggregationRange = 1440; + private String _usageTimeZone = "GMT"; + private final long mgmtSrvrId = MacAddress.getMacAddress().toLong(); + private static final int ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_COOPERATION = 5; // 5 seconds + + ScheduledExecutorService _executor; + ScheduledExecutorService _checkExecutor; + ScheduledExecutorService _networkStatsUpdateExecutor; + + Account _systemAcct; + + @Override + public boolean sendSshKeysToHost(Long hostId, String pubKey, String prvKey) { + ModifySshKeysCommand cmd = new ModifySshKeysCommand(pubKey, prvKey); + final Answer answer = _agentMgr.easySend(hostId, cmd); + + if (answer != null) { + return true; + } else { + return false; + } + } + + @Override + public VirtualRouter destroyRouter(final long routerId) throws ResourceUnavailableException, ConcurrentOperationException { + UserContext context = UserContext.current(); + User user = _accountMgr.getActiveUser(context.getCallerUserId()); + + if (s_logger.isDebugEnabled()) { + s_logger.debug("Attempting to destroy router " + routerId); + } + + DomainRouterVO router = _routerDao.findById(routerId); + if (router == null) { + return null; + } + + _accountMgr.checkAccess(context.getCaller(), null, true, router); + + boolean result = _itMgr.expunge(router, user, _accountMgr.getAccount(router.getAccountId())); + + if (result) { + return router; + } + return null; + } + + @Override + @DB + public VirtualRouter upgradeRouter(UpgradeRouterCmd cmd) { + Long routerId = cmd.getId(); + Long serviceOfferingId = cmd.getServiceOfferingId(); + Account caller = UserContext.current().getCaller(); + + DomainRouterVO router = _routerDao.findById(routerId); + if (router == null) { + throw new InvalidParameterValueException("Unable to find router with id " + routerId); + } + + _accountMgr.checkAccess(caller, null, true, router); + + if (router.getServiceOfferingId() == serviceOfferingId) { + s_logger.debug("Router: " + routerId + "already has service offering: " + serviceOfferingId); + return _routerDao.findById(routerId); + } + + ServiceOffering newServiceOffering = _configMgr.getServiceOffering(serviceOfferingId); + if (newServiceOffering == null) { + throw new InvalidParameterValueException("Unable to find service offering with id " + serviceOfferingId); + } + + // check if it is a system service offering, if yes return with error as it cannot be used for user vms + if (!newServiceOffering.getSystemUse()) { + throw new InvalidParameterValueException("Cannot upgrade router vm to a non system service offering " + serviceOfferingId); + } + + // Check that the router is stopped + if (!router.getState().equals(State.Stopped)) { + s_logger.warn("Unable to upgrade router " + router.toString() + " in state " + router.getState()); + throw new InvalidParameterValueException("Unable to upgrade router " + router.toString() + " in state " + router.getState() + + "; make sure the router is stopped and not in an error state before upgrading."); + } + + ServiceOfferingVO currentServiceOffering = _serviceOfferingDao.findById(router.getServiceOfferingId()); + + // Check that the service offering being upgraded to has the same storage pool preference as the VM's current service + // offering + if (currentServiceOffering.getUseLocalStorage() != newServiceOffering.getUseLocalStorage()) { + throw new InvalidParameterValueException("Can't upgrade, due to new local storage status : " + + newServiceOffering.getUseLocalStorage() + " is different from " + + "curruent local storage status: " + currentServiceOffering.getUseLocalStorage()); + } + + router.setServiceOfferingId(serviceOfferingId); + if (_routerDao.update(routerId, router)) { + return _routerDao.findById(routerId); + } else { + throw new CloudRuntimeException("Unable to upgrade router " + routerId); + } + + } + + @Override + public boolean savePasswordToRouter(Network network, NicProfile nic, VirtualMachineProfile profile, List routers) throws ResourceUnavailableException { + if (routers == null || routers.isEmpty()) { + s_logger.warn("Unable save password, router doesn't exist in network " + network.getId()); + throw new CloudRuntimeException("Unable to save password to router"); + } + + UserVm userVm = profile.getVirtualMachine(); + String password = (String) profile.getParameter(Param.VmPassword); + String encodedPassword = PasswordGenerator.rot13(password); + DataCenter dc = _dcDao.findById(userVm.getDataCenterIdToDeployIn()); + + boolean result = true; + for (VirtualRouter router : routers) { + boolean sendPassword = true; + if (dc.getNetworkType() == NetworkType.Basic && userVm.getPodIdToDeployIn().longValue() != router.getPodIdToDeployIn().longValue()) { + sendPassword = false; + } + + if (sendPassword) { + Commands cmds = new Commands(OnError.Continue); + SavePasswordCommand cmd = new SavePasswordCommand(encodedPassword, nic.getIp4Address(), userVm.getHostName()); + cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); + cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); + DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); + cmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); + cmds.addCommand("password", cmd); + + result = result && sendCommandsToRouter(router, cmds); + } + } + return result; + } + + @Override @ActionEvent(eventType = EventTypes.EVENT_ROUTER_STOP, eventDescription = "stopping router Vm", async = true) + public VirtualRouter stopRouter(long routerId, boolean forced) throws ResourceUnavailableException, ConcurrentOperationException { + UserContext context = UserContext.current(); + Account account = context.getCaller(); + + // verify parameters + DomainRouterVO router = _routerDao.findById(routerId); + if (router == null) { + throw new InvalidParameterValueException("Unable to find router by id " + routerId + "."); + } + + _accountMgr.checkAccess(account, null, true, router); + + UserVO user = _userDao.findById(UserContext.current().getCallerUserId()); + + VirtualRouter virtualRouter = stop(router, forced, user, account); + if(virtualRouter == null){ + throw new CloudRuntimeException("Failed to stop router with id " + routerId); + } + return virtualRouter; + } + + @DB + public void processStopOrRebootAnswer(final DomainRouterVO router, Answer answer) { + final Transaction txn = Transaction.currentTxn(); + try { + txn.start(); + //FIXME!!! - UserStats command should grab bytesSent/Received for all guest interfaces of the VR + List routerGuestNtwkIds = _routerDao.getRouterNetworks(router.getId()); + for (Long guestNtwkId : routerGuestNtwkIds) { + final UserStatisticsVO userStats = _userStatsDao.lock(router.getAccountId(), router.getDataCenterIdToDeployIn(), + guestNtwkId, null, router.getId(), router.getType().toString()); + if (userStats != null) { + final long currentBytesRcvd = userStats.getCurrentBytesReceived(); + userStats.setCurrentBytesReceived(0); + userStats.setNetBytesReceived(userStats.getNetBytesReceived() + currentBytesRcvd); + + final long currentBytesSent = userStats.getCurrentBytesSent(); + userStats.setCurrentBytesSent(0); + userStats.setNetBytesSent(userStats.getNetBytesSent() + currentBytesSent); + _userStatsDao.update(userStats.getId(), userStats); + s_logger.debug("Successfully updated user statistics as a part of domR " + router + " reboot/stop"); + } else { + s_logger.warn("User stats were not created for account " + router.getAccountId() + " and dc " + router.getDataCenterIdToDeployIn()); + } + } + + txn.commit(); + } catch (final Exception e) { + txn.rollback(); + throw new CloudRuntimeException("Problem updating stats after reboot/stop ", e); + } + } + + @Override @ActionEvent(eventType = EventTypes.EVENT_ROUTER_REBOOT, eventDescription = "rebooting router Vm", async = true) + public VirtualRouter rebootRouter(long routerId, boolean reprogramNetwork) throws ConcurrentOperationException, + ResourceUnavailableException, InsufficientCapacityException { + Account caller = UserContext.current().getCaller(); + + // verify parameters + DomainRouterVO router = _routerDao.findById(routerId); + if (router == null) { + throw new InvalidParameterValueException("Unable to find domain router with id " + routerId + "."); + } + + _accountMgr.checkAccess(caller, null, true, router); + + // Can reboot domain router only in Running state + if (router == null || router.getState() != State.Running) { + s_logger.warn("Unable to reboot, virtual router is not in the right state " + router.getState()); + throw new ResourceUnavailableException("Unable to reboot domR, it is not in right state " + router.getState(), + DataCenter.class, router.getDataCenterIdToDeployIn()); + } + + UserVO user = _userDao.findById(UserContext.current().getCallerUserId()); + s_logger.debug("Stopping and starting router " + router + " as a part of router reboot"); + + if (stop(router, false, user, caller) != null) { + return startRouter(routerId, reprogramNetwork); + } else { + throw new CloudRuntimeException("Failed to reboot router " + router); + } + } + + @Override + public boolean configure(final String name, final Map params) throws ConfigurationException { + _name = name; + + _executor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("RouterMonitor")); + _checkExecutor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("RouterStatusMonitor")); + _networkStatsUpdateExecutor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("NetworkStatsUpdater")); + + final ComponentLocator locator = ComponentLocator.getCurrentLocator(); + + final Map configs = _configDao.getConfiguration("AgentManager", params); + + _mgmt_host = configs.get("host"); + _routerRamSize = NumbersUtil.parseInt(configs.get("router.ram.size"), DEFAULT_ROUTER_VM_RAMSIZE); + _routerCpuMHz = NumbersUtil.parseInt(configs.get("router.cpu.mhz"), DEFAULT_ROUTER_CPU_MHZ); + + _routerExtraPublicNics = NumbersUtil.parseInt(_configDao.getValue(Config.RouterExtraPublicNics.key()), 2); + + String value = configs.get("start.retry"); + _retry = NumbersUtil.parseInt(value, 2); + + value = configs.get("router.stats.interval"); + _routerStatsInterval = NumbersUtil.parseInt(value, 300); + + value = configs.get("router.check.interval"); + _routerCheckInterval = NumbersUtil.parseInt(value, 30); + + _instance = configs.get("instance.name"); + if (_instance == null) { + _instance = "DEFAULT"; + } + + String rpValue = configs.get("network.disable.rpfilter"); + if (rpValue != null && rpValue.equalsIgnoreCase("true")) { + _disable_rp_filter = true; + } + + _dnsBasicZoneUpdates = String.valueOf(_configDao.getValue(Config.DnsBasicZoneUpdates.key())); + + s_logger.info("Router configurations: " + "ramsize=" + _routerRamSize); + + final UserStatisticsDao statsDao = locator.getDao(UserStatisticsDao.class); + if (statsDao == null) { + throw new ConfigurationException("Unable to get " + UserStatisticsDao.class.getName()); + } + + _agentMgr.registerForHostEvents(new SshKeysDistriMonitor(_agentMgr, _hostDao, _configDao), true, false, false); + _itMgr.registerGuru(VirtualMachine.Type.DomainRouter, this); + + boolean useLocalStorage = Boolean.parseBoolean(configs.get(Config.SystemVMUseLocalStorage.key())); + _offering = new ServiceOfferingVO("System Offering For Software Router", 1, _routerRamSize, _routerCpuMHz, null, + null, true, null, useLocalStorage, true, null, true, VirtualMachine.Type.DomainRouter, true); + _offering.setUniqueName(ServiceOffering.routerDefaultOffUniqueName); + _offering = _serviceOfferingDao.persistSystemServiceOffering(_offering); + + // this can sometimes happen, if DB is manually or programmatically manipulated + if(_offering == null) { + String msg = "Data integrity problem : System Offering For Software router VM has been removed?"; + s_logger.error(msg); + throw new ConfigurationException(msg); + } + + _systemAcct = _accountMgr.getSystemAccount(); + + String aggregationRange = configs.get("usage.stats.job.aggregation.range"); + _usageAggregationRange = NumbersUtil.parseInt(aggregationRange, 1440); + _usageTimeZone = configs.get("usage.aggregation.timezone"); + if(_usageTimeZone == null){ + _usageTimeZone = "GMT"; + } + + _agentMgr.registerForHostEvents(this, true, false, false); + + s_logger.info("DomainRouterManager is configured."); + + return true; + } + + @Override + public String getName() { + return _name; + } + + @Override + public boolean start() { + if (_routerStatsInterval > 0){ + _executor.scheduleAtFixedRate(new NetworkUsageTask(), _routerStatsInterval, _routerStatsInterval, TimeUnit.SECONDS); + }else{ + s_logger.debug("router.stats.interval - " + _routerStatsInterval+ " so not scheduling the router stats thread"); + } + + //Schedule Network stats update task + TimeZone usageTimezone = TimeZone.getTimeZone(_usageTimeZone); + Calendar cal = Calendar.getInstance(usageTimezone); + cal.setTime(new Date()); + long endDate = 0; + int HOURLY_TIME = 60; + final int DAILY_TIME = 60 * 24; + if (_usageAggregationRange == DAILY_TIME) { + cal.roll(Calendar.DAY_OF_YEAR, false); + cal.set(Calendar.HOUR_OF_DAY, 0); + cal.set(Calendar.MINUTE, 0); + cal.set(Calendar.SECOND, 0); + cal.set(Calendar.MILLISECOND, 0); + cal.roll(Calendar.DAY_OF_YEAR, true); + cal.add(Calendar.MILLISECOND, -1); + endDate = cal.getTime().getTime(); + } else if (_usageAggregationRange == HOURLY_TIME) { + cal.roll(Calendar.HOUR_OF_DAY, false); + cal.set(Calendar.MINUTE, 0); + cal.set(Calendar.SECOND, 0); + cal.set(Calendar.MILLISECOND, 0); + cal.roll(Calendar.HOUR_OF_DAY, true); + cal.add(Calendar.MILLISECOND, -1); + endDate = cal.getTime().getTime(); + } else { + endDate = cal.getTime().getTime(); + } + + _networkStatsUpdateExecutor.scheduleAtFixedRate(new NetworkStatsUpdateTask(), (endDate - System.currentTimeMillis()), + (_usageAggregationRange * 60 * 1000), TimeUnit.MILLISECONDS); + + if (_routerCheckInterval > 0) { + _checkExecutor.scheduleAtFixedRate(new CheckRouterTask(), _routerCheckInterval, _routerCheckInterval, TimeUnit.SECONDS); + } else { + s_logger.debug("router.check.interval - " + _routerCheckInterval+ " so not scheduling the redundant router checking thread"); + } + + return true; + } + + @Override + public boolean stop() { + return true; + } + + protected VirtualNetworkApplianceManagerImpl() { + } + + @Override + public Long convertToId(final String vmName) { + if (!VirtualMachineName.isValidRouterName(vmName, _instance)) { + return null; + } + + return VirtualMachineName.getRouterId(vmName); + } + + private VmDataCommand generateVmDataCommand(VirtualRouter router, String vmPrivateIpAddress, String userData, + String serviceOffering, String zoneName, String guestIpAddress, String vmName, + String vmInstanceName, long vmId, String publicKey, long guestNetworkId) { + VmDataCommand cmd = new VmDataCommand(vmPrivateIpAddress, vmName); + + cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); + cmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(guestNetworkId, router.getId())); + cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); + + DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); + cmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); + + cmd.addVmData("userdata", "user-data", userData); + cmd.addVmData("metadata", "service-offering", StringUtils.unicodeEscape(serviceOffering)); + cmd.addVmData("metadata", "availability-zone", StringUtils.unicodeEscape(zoneName)); + cmd.addVmData("metadata", "local-ipv4", guestIpAddress); + cmd.addVmData("metadata", "local-hostname", StringUtils.unicodeEscape(vmName)); + if (dcVo.getNetworkType() == NetworkType.Basic) { + cmd.addVmData("metadata", "public-ipv4", guestIpAddress); + cmd.addVmData("metadata", "public-hostname", StringUtils.unicodeEscape(vmName)); + } else + { + if (router.getPublicIpAddress() == null) { + cmd.addVmData("metadata", "public-ipv4", guestIpAddress); + } else { + cmd.addVmData("metadata", "public-ipv4", router.getPublicIpAddress()); } - } - } - - protected class NetworkStatsUpdateTask implements Runnable { - - public NetworkStatsUpdateTask() { - } - - @Override - public void run() { - GlobalLock scanLock = GlobalLock.getInternLock("network.stats"); - try { - if(scanLock.lock(ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_COOPERATION)) { - //Check for ownership - //msHost in UP state with min id should run the job - ManagementServerHostVO msHost = _msHostDao.findOneInUpState(new Filter(ManagementServerHostVO.class, "id", true, 0L, 1L)); - if(msHost == null || (msHost.getMsid() != mgmtSrvrId)){ - s_logger.debug("Skipping aggregate network stats update"); - scanLock.unlock(); - return; - } - Transaction txn = Transaction.open(Transaction.CLOUD_DB); - try { - txn.start(); - //get all stats with delta > 0 - List updatedStats = _statsDao.listUpdatedStats(); - Date updatedTime = new Date(); - for(UserStatisticsVO stat : updatedStats){ - //update agg bytes - stat.setAggBytesReceived(stat.getCurrentBytesReceived() + stat.getNetBytesReceived()); - stat.setAggBytesSent(stat.getCurrentBytesSent() + stat.getNetBytesSent()); - _userStatsDao.update(stat.getId(), stat); - //insert into op_user_stats_log - UserStatsLogVO statsLog = new UserStatsLogVO(stat.getId(), stat.getNetBytesReceived(), stat.getNetBytesSent(), stat.getCurrentBytesReceived(), - stat.getCurrentBytesSent(), stat.getAggBytesReceived(), stat.getAggBytesSent(), updatedTime); - _userStatsLogDao.persist(statsLog); - } - s_logger.debug("Successfully updated aggregate network stats"); - txn.commit(); - } catch (Exception e){ - txn.rollback(); - s_logger.debug("Failed to update aggregate network stats", e); - } finally { - scanLock.unlock(); - txn.close(); - } - } - } catch (Exception e){ - s_logger.debug("Exception while trying to acquire network stats lock", e); - } finally { - scanLock.releaseRef(); - } - } - } - - @DB - protected void updateSite2SiteVpnConnectionState(List routers) { - for (DomainRouterVO router : routers) { - List conns = _s2sVpnMgr.getConnectionsForRouter(router); - if (conns == null || conns.isEmpty()) { - continue; - } - if (router.getState() != State.Running) { - for (Site2SiteVpnConnectionVO conn : conns) { - if (conn.getState() != Site2SiteVpnConnection.State.Error) { - conn.setState(Site2SiteVpnConnection.State.Disconnected); - _s2sVpnConnectionDao.persist(conn); - } - } - continue; - } - List ipList = new ArrayList(); - for (Site2SiteVpnConnectionVO conn : conns) { - if (conn.getState() != Site2SiteVpnConnection.State.Connected && - conn.getState() != Site2SiteVpnConnection.State.Disconnected) { - continue; - } - Site2SiteCustomerGateway gw = _s2sCustomerGatewayDao.findById(conn.getCustomerGatewayId()); - ipList.add(gw.getGatewayIp()); - } - String privateIP = router.getPrivateIpAddress(); - HostVO host = _hostDao.findById(router.getHostId()); - if (host == null || host.getStatus() != Status.Up) { - continue; - } else if (host.getManagementServerId() != ManagementServerNode.getManagementServerId()) { - /* Only cover hosts managed by this management server */ - continue; - } else if (privateIP != null) { - final CheckS2SVpnConnectionsCommand command = new CheckS2SVpnConnectionsCommand(ipList); - command.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); - command.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); - command.setWait(30); - final Answer origAnswer = _agentMgr.easySend(router.getHostId(), command); - CheckS2SVpnConnectionsAnswer answer = null; - if (origAnswer instanceof CheckS2SVpnConnectionsAnswer) { - answer = (CheckS2SVpnConnectionsAnswer)origAnswer; - } else { - s_logger.warn("Unable to update router " + router.getHostName() + "'s VPN connection status"); - continue; - } - if (!answer.getResult()) { - s_logger.warn("Unable to update router " + router.getHostName() + "'s VPN connection status"); - continue; - } - for (Site2SiteVpnConnectionVO conn : conns) { + cmd.addVmData("metadata", "public-hostname", router.getPublicIpAddress()); + } + cmd.addVmData("metadata", "instance-id", vmInstanceName); + cmd.addVmData("metadata", "vm-id", String.valueOf(vmId)); + cmd.addVmData("metadata", "public-keys", publicKey); + + String cloudIdentifier = _configDao.getValue("cloud.identifier"); + if (cloudIdentifier == null) { + cloudIdentifier = ""; + } else { + cloudIdentifier = "CloudStack-{" + cloudIdentifier + "}"; + } + cmd.addVmData("metadata", "cloud-identifier", cloudIdentifier); + + return cmd; + } + + protected class NetworkUsageTask implements Runnable { + + public NetworkUsageTask() { + } + + @Override + public void run() { + try{ + final List routers = _routerDao.listByStateAndNetworkType(State.Running, GuestType.Isolated, mgmtSrvrId); + s_logger.debug("Found " + routers.size() + " running routers. "); + + for (DomainRouterVO router : routers) { + String privateIP = router.getPrivateIpAddress(); + + if (privateIP != null) { + List routerNics = _nicDao.listByVmId(router.getId()); + for (Nic routerNic : routerNics) { + Network network = _networkMgr.getNetwork(routerNic.getNetworkId()); + if (network.getTrafficType() == TrafficType.Public) { + boolean forVpc = router.getVpcId() != null; + final NetworkUsageCommand usageCmd = new NetworkUsageCommand(privateIP, router.getHostName(), + forVpc, routerNic.getIp4Address()); + UserStatisticsVO previousStats = _statsDao.findBy(router.getAccountId(), + router.getDataCenterIdToDeployIn(), network.getId(), null, router.getId(), router.getType().toString()); + NetworkUsageAnswer answer = null; + try { + answer = (NetworkUsageAnswer) _agentMgr.easySend(router.getHostId(), usageCmd); + } catch (Exception e) { + s_logger.warn("Error while collecting network stats from router: "+router.getInstanceName()+" from host: "+router.getHostId(), e); + continue; + } + + if (answer != null) { + if (!answer.getResult()) { + s_logger.warn("Error while collecting network stats from router: "+router.getInstanceName()+" from host: "+router.getHostId() + "; details: " + answer.getDetails()); + continue; + } + Transaction txn = Transaction.open(Transaction.CLOUD_DB); + try { + if ((answer.getBytesReceived() == 0) && (answer.getBytesSent() == 0)) { + s_logger.debug("Recieved and Sent bytes are both 0. Not updating user_statistics"); + continue; + } + txn.start(); + UserStatisticsVO stats = _statsDao.lock(router.getAccountId(), + router.getDataCenterIdToDeployIn(), network.getId(), routerNic.getIp4Address(), router.getId(), router.getType().toString()); + if (stats == null) { + s_logger.warn("unable to find stats for account: " + router.getAccountId()); + continue; + } + + if(previousStats != null + && ((previousStats.getCurrentBytesReceived() != stats.getCurrentBytesReceived()) + || (previousStats.getCurrentBytesSent() != stats.getCurrentBytesSent()))){ + s_logger.debug("Router stats changed from the time NetworkUsageCommand was sent. " + + "Ignoring current answer. Router: "+answer.getRouterName()+" Rcvd: " + + answer.getBytesReceived()+ "Sent: " +answer.getBytesSent()); + continue; + } + + if (stats.getCurrentBytesReceived() > answer.getBytesReceived()) { + if (s_logger.isDebugEnabled()) { + s_logger.debug("Received # of bytes that's less than the last one. " + + "Assuming something went wrong and persisting it. Router: " + + answer.getRouterName()+" Reported: " + answer.getBytesReceived() + + " Stored: " + stats.getCurrentBytesReceived()); + } + stats.setNetBytesReceived(stats.getNetBytesReceived() + stats.getCurrentBytesReceived()); + } + stats.setCurrentBytesReceived(answer.getBytesReceived()); + if (stats.getCurrentBytesSent() > answer.getBytesSent()) { + if (s_logger.isDebugEnabled()) { + s_logger.debug("Received # of bytes that's less than the last one. " + + "Assuming something went wrong and persisting it. Router: " + + answer.getRouterName()+" Reported: " + answer.getBytesSent() + + " Stored: " + stats.getCurrentBytesSent()); + } + stats.setNetBytesSent(stats.getNetBytesSent() + stats.getCurrentBytesSent()); + } + stats.setCurrentBytesSent(answer.getBytesSent()); + _statsDao.update(stats.getId(), stats); + txn.commit(); + } catch (Exception e) { + txn.rollback(); + s_logger.warn("Unable to update user statistics for account: " + router.getAccountId() + + " Rx: " + answer.getBytesReceived() + "; Tx: " + answer.getBytesSent()); + } finally { + txn.close(); + } + } + } + } + } + } + } catch (Exception e) { + s_logger.warn("Error while collecting network stats", e); + } + } + } + + protected class NetworkStatsUpdateTask implements Runnable { + + public NetworkStatsUpdateTask() { + } + + @Override + public void run() { + GlobalLock scanLock = GlobalLock.getInternLock("network.stats"); + try { + if(scanLock.lock(ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_COOPERATION)) { + //Check for ownership + //msHost in UP state with min id should run the job + ManagementServerHostVO msHost = _msHostDao.findOneInUpState(new Filter(ManagementServerHostVO.class, "id", true, 0L, 1L)); + if(msHost == null || (msHost.getMsid() != mgmtSrvrId)){ + s_logger.debug("Skipping aggregate network stats update"); + scanLock.unlock(); + return; + } + Transaction txn = Transaction.open(Transaction.CLOUD_DB); + try { + txn.start(); + //get all stats with delta > 0 + List updatedStats = _statsDao.listUpdatedStats(); + Date updatedTime = new Date(); + for(UserStatisticsVO stat : updatedStats){ + //update agg bytes + stat.setAggBytesReceived(stat.getCurrentBytesReceived() + stat.getNetBytesReceived()); + stat.setAggBytesSent(stat.getCurrentBytesSent() + stat.getNetBytesSent()); + _userStatsDao.update(stat.getId(), stat); + //insert into op_user_stats_log + UserStatsLogVO statsLog = new UserStatsLogVO(stat.getId(), stat.getNetBytesReceived(), stat.getNetBytesSent(), stat.getCurrentBytesReceived(), + stat.getCurrentBytesSent(), stat.getAggBytesReceived(), stat.getAggBytesSent(), updatedTime); + _userStatsLogDao.persist(statsLog); + } + s_logger.debug("Successfully updated aggregate network stats"); + txn.commit(); + } catch (Exception e){ + txn.rollback(); + s_logger.debug("Failed to update aggregate network stats", e); + } finally { + scanLock.unlock(); + txn.close(); + } + } + } catch (Exception e){ + s_logger.debug("Exception while trying to acquire network stats lock", e); + } finally { + scanLock.releaseRef(); + } + } + } + + @DB + protected void updateSite2SiteVpnConnectionState(List routers) { + for (DomainRouterVO router : routers) { + List conns = _s2sVpnMgr.getConnectionsForRouter(router); + if (conns == null || conns.isEmpty()) { + continue; + } + if (router.getState() != State.Running) { + for (Site2SiteVpnConnectionVO conn : conns) { + if (conn.getState() != Site2SiteVpnConnection.State.Error) { + conn.setState(Site2SiteVpnConnection.State.Disconnected); + _s2sVpnConnectionDao.persist(conn); + } + } + continue; + } + List ipList = new ArrayList(); + for (Site2SiteVpnConnectionVO conn : conns) { + if (conn.getState() != Site2SiteVpnConnection.State.Connected && + conn.getState() != Site2SiteVpnConnection.State.Disconnected) { + continue; + } + Site2SiteCustomerGateway gw = _s2sCustomerGatewayDao.findById(conn.getCustomerGatewayId()); + ipList.add(gw.getGatewayIp()); + } + String privateIP = router.getPrivateIpAddress(); + HostVO host = _hostDao.findById(router.getHostId()); + if (host == null || host.getStatus() != Status.Up) { + continue; + } else if (host.getManagementServerId() != ManagementServerNode.getManagementServerId()) { + /* Only cover hosts managed by this management server */ + continue; + } else if (privateIP != null) { + final CheckS2SVpnConnectionsCommand command = new CheckS2SVpnConnectionsCommand(ipList); + command.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); + command.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); + command.setWait(30); + final Answer origAnswer = _agentMgr.easySend(router.getHostId(), command); + CheckS2SVpnConnectionsAnswer answer = null; + if (origAnswer instanceof CheckS2SVpnConnectionsAnswer) { + answer = (CheckS2SVpnConnectionsAnswer)origAnswer; + } else { + s_logger.warn("Unable to update router " + router.getHostName() + "'s VPN connection status"); + continue; + } + if (!answer.getResult()) { + s_logger.warn("Unable to update router " + router.getHostName() + "'s VPN connection status"); + continue; + } + for (Site2SiteVpnConnectionVO conn : conns) { Site2SiteVpnConnectionVO lock = _s2sVpnConnectionDao.acquireInLockTable(conn.getId()); if (lock == null) { throw new CloudRuntimeException("Unable to acquire lock on " + lock); - } + } try { - if (conn.getState() != Site2SiteVpnConnection.State.Connected && - conn.getState() != Site2SiteVpnConnection.State.Disconnected) { - continue; - } - Site2SiteVpnConnection.State oldState = conn.getState(); - Site2SiteCustomerGateway gw = _s2sCustomerGatewayDao.findById(conn.getCustomerGatewayId()); - if (answer.isConnected(gw.getGatewayIp())) { - conn.setState(Site2SiteVpnConnection.State.Connected); - } else { - conn.setState(Site2SiteVpnConnection.State.Disconnected); - } - _s2sVpnConnectionDao.persist(conn); - if (oldState != conn.getState()) { - String title = "Site-to-site Vpn Connection to " + gw.getName() + - " just switch from " + oldState + " to " + conn.getState(); - String context = "Site-to-site Vpn Connection to " + gw.getName() + " on router " + router.getHostName() + - "(id: " + router.getId() + ") " + " just switch from " + oldState + " to " + conn.getState(); - s_logger.info(context); - _alertMgr.sendAlert(AlertManager.ALERT_TYPE_DOMAIN_ROUTER, - router.getDataCenterIdToDeployIn(), router.getPodIdToDeployIn(), title, context); - } - } finally { - _s2sVpnConnectionDao.releaseFromLockTable(lock.getId()); - } - } - } - } - } - - protected void updateRoutersRedundantState(List routers) { - boolean updated = false; - for (DomainRouterVO router : routers) { - updated = false; - if (!router.getIsRedundantRouter()) { - continue; - } - RedundantState prevState = router.getRedundantState(); - if (router.getState() != State.Running) { - router.setRedundantState(RedundantState.UNKNOWN); - router.setIsPriorityBumpUp(false); - updated = true; - } else { - String privateIP = router.getPrivateIpAddress(); - HostVO host = _hostDao.findById(router.getHostId()); - if (host == null || host.getStatus() != Status.Up) { - router.setRedundantState(RedundantState.UNKNOWN); - updated = true; - } else if (host.getManagementServerId() != ManagementServerNode.getManagementServerId()) { - /* Only cover hosts managed by this management server */ - continue; - } else if (privateIP != null) { - final CheckRouterCommand command = new CheckRouterCommand(); - command.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); - command.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); - command.setWait(60); - final Answer origAnswer = _agentMgr.easySend(router.getHostId(), command); - CheckRouterAnswer answer = null; - if (origAnswer instanceof CheckRouterAnswer) { - answer = (CheckRouterAnswer)origAnswer; - } else { - s_logger.warn("Unable to update router " + router.getHostName() + "'s status"); - } - RedundantState state = RedundantState.UNKNOWN; - boolean isBumped = router.getIsPriorityBumpUp(); - if (answer != null && answer.getResult()) { - state = answer.getState(); - isBumped = answer.isBumped(); - } - router.setRedundantState(state); - router.setIsPriorityBumpUp(isBumped); - updated = true; - } - } - if (updated) { - Transaction txn = Transaction.open(Transaction.CLOUD_DB); - try { - txn.start(); - _routerDao.update(router.getId(), router); - txn.commit(); - } catch (Exception e) { - txn.rollback(); - s_logger.warn("Unable to update router status for account: " + router.getAccountId()); - } finally { - txn.close(); - } - } - RedundantState currState = router.getRedundantState(); - if (prevState != currState) { - String title = "Redundant virtual router " + router.getInstanceName() + - " just switch from " + prevState + " to " + currState; - String context = "Redundant virtual router (name: " + router.getHostName() + ", id: " + router.getId() + ") " + - " just switch from " + prevState + " to " + currState; - s_logger.info(context); - if (currState == RedundantState.MASTER) { - _alertMgr.sendAlert(AlertManager.ALERT_TYPE_DOMAIN_ROUTER, - router.getDataCenterIdToDeployIn(), router.getPodIdToDeployIn(), title, context); - } - } - } - } - - //Ensure router status is update to date before execute this function. The function would try best to recover all routers except MASTER - protected void recoverRedundantNetwork(DomainRouterVO masterRouter, DomainRouterVO backupRouter) { - UserContext context = UserContext.current(); - context.setAccountId(1); - if (masterRouter.getState() == State.Running && backupRouter.getState() == State.Running) { - HostVO masterHost = _hostDao.findById(masterRouter.getHostId()); - HostVO backupHost = _hostDao.findById(backupRouter.getHostId()); - if (masterHost.getStatus() == Status.Up && backupHost.getStatus() == Status.Up) { - String title = "Reboot " + backupRouter.getInstanceName() + " to ensure redundant virtual routers work"; - if (s_logger.isDebugEnabled()) { - s_logger.debug(title); - } - _alertMgr.sendAlert(AlertManager.ALERT_TYPE_DOMAIN_ROUTER, - backupRouter.getDataCenterIdToDeployIn(), backupRouter.getPodIdToDeployIn(), title, title); - try { - rebootRouter(backupRouter.getId(), false); - } catch (ConcurrentOperationException e) { - s_logger.warn("Fail to reboot " + backupRouter.getInstanceName(), e); - } catch (ResourceUnavailableException e) { - s_logger.warn("Fail to reboot " + backupRouter.getInstanceName(), e); - } catch (InsufficientCapacityException e) { - s_logger.warn("Fail to reboot " + backupRouter.getInstanceName(), e); - } - } - } - } - - private int getRealPriority(DomainRouterVO router) { - int priority = router.getPriority(); - if (router.getIsPriorityBumpUp()) { - priority += DEFAULT_DELTA; - } - return priority; - } - - protected class CheckRouterTask implements Runnable { - - public CheckRouterTask() { - } - - /* - * In order to make fail-over works well at any time, we have to ensure: - * 1. Backup router's priority = Master's priority - DELTA + 1 - * 2. Backup router's priority hasn't been bumped up. - */ - private void checkSanity(List routers) { - Set checkedNetwork = new HashSet(); - for (DomainRouterVO router : routers) { - if (!router.getIsRedundantRouter()) { - continue; - } - - List routerGuestNtwkIds = _routerDao.getRouterNetworks(router.getId()); - - for (Long routerGuestNtwkId : routerGuestNtwkIds) { - if (checkedNetwork.contains(routerGuestNtwkId)) { - continue; - } - checkedNetwork.add(routerGuestNtwkId); - List checkingRouters = _routerDao.listByNetworkAndRole(routerGuestNtwkId, Role.VIRTUAL_ROUTER); - if (checkingRouters.size() != 2) { - continue; - } - DomainRouterVO masterRouter = null; - DomainRouterVO backupRouter = null; - for (DomainRouterVO r : checkingRouters) { - if (r.getRedundantState() == RedundantState.MASTER) { - if (masterRouter == null) { - masterRouter = r; - } else { - //Duplicate master! We give up, until the admin fix duplicate MASTER issue - break; - } - } else if (r.getRedundantState() == RedundantState.BACKUP) { - if (backupRouter == null) { - backupRouter = r; - } else { - break; - } - } - } - if (masterRouter != null && backupRouter != null) { - if (getRealPriority(masterRouter) - DEFAULT_DELTA + 1 != getRealPriority(backupRouter) || backupRouter.getIsPriorityBumpUp()) { - recoverRedundantNetwork(masterRouter, backupRouter); - } - } - } - } - } - - private void checkDuplicateMaster(List routers) { - Map networkRouterMaps = new HashMap(); - for (DomainRouterVO router : routers) { - List routerGuestNtwkIds = _routerDao.getRouterNetworks(router.getId()); - - for (Long routerGuestNtwkId : routerGuestNtwkIds) { - if (router.getRedundantState() == RedundantState.MASTER) { - if (networkRouterMaps.containsKey(routerGuestNtwkId)) { - DomainRouterVO dupRouter = networkRouterMaps.get(routerGuestNtwkId); - String title = "More than one redundant virtual router is in MASTER state! Router " + router.getHostName() + " and router " + dupRouter.getHostName(); - String context = "Virtual router (name: " + router.getHostName() + ", id: " + router.getId() + " and router (name: " - + dupRouter.getHostName() + ", id: " + router.getId() + ") are both in MASTER state! If the problem persist, restart both of routers. "; - - _alertMgr.sendAlert(AlertManager.ALERT_TYPE_DOMAIN_ROUTER, router.getDataCenterIdToDeployIn(), router.getPodIdToDeployIn(), title, context); - _alertMgr.sendAlert(AlertManager.ALERT_TYPE_DOMAIN_ROUTER, dupRouter.getDataCenterIdToDeployIn(), dupRouter.getPodIdToDeployIn(), title, context); - } else { - networkRouterMaps.put(routerGuestNtwkId, router); - } - } - } - } - } - - @Override - public void run() { - try { - final List routers = _routerDao.listIsolatedByHostId(null); - s_logger.debug("Found " + routers.size() + " routers. "); - - updateRoutersRedundantState(routers); - updateSite2SiteVpnConnectionState(routers); - - /* FIXME assumed the a pair of redundant routers managed by same mgmt server, - * then the update above can get the latest status */ - checkDuplicateMaster(routers); - checkSanity(routers); - } catch (Exception ex) { - s_logger.error("Fail to complete the CheckRouterTask! ", ex); - } - } - } - - - private final int DEFAULT_PRIORITY = 100; - private final int DEFAULT_DELTA = 2; - - protected int getUpdatedPriority(Network guestNetwork, List routers, DomainRouterVO exclude) throws InsufficientVirtualNetworkCapcityException { - int priority; - if (routers.size() == 0) { - priority = DEFAULT_PRIORITY; - } else { - int maxPriority = 0; - for (DomainRouterVO r : routers) { - if (!r.getIsRedundantRouter()) { - throw new CloudRuntimeException("Redundant router is mixed with single router in one network!"); - } - //FIXME Assume the maxPriority one should be running or just created. - if (r.getId() != exclude.getId() && getRealPriority(r) > maxPriority) { - maxPriority = getRealPriority(r); - } - } - if (maxPriority == 0) { - return DEFAULT_PRIORITY; - } - if (maxPriority < 20) { - s_logger.error("Current maximum priority is too low!"); - throw new InsufficientVirtualNetworkCapcityException("Current maximum priority is too low as " + maxPriority + "!", - guestNetwork.getId()); - } else if (maxPriority > 200) { - s_logger.error("Too many times fail-over happened! Current maximum priority is too high as " + maxPriority + "!"); - throw new InsufficientVirtualNetworkCapcityException("Too many times fail-over happened! Current maximum priority is too high as " - + maxPriority + "!", guestNetwork.getId()); - } - priority = maxPriority - DEFAULT_DELTA + 1; - } - return priority; - } - - /* - * Ovm won't support any system. So we have to choose a partner cluster in the same pod to start domain router for us - */ - private HypervisorType getClusterToStartDomainRouterForOvm(long podId) { - List clusters = _clusterDao.listByPodId(podId); - for (ClusterVO cv : clusters) { - if (cv.getHypervisorType() == HypervisorType.Ovm || cv.getHypervisorType() == HypervisorType.BareMetal) { - continue; - } - - List hosts = _resourceMgr.listAllHostsInCluster(cv.getId()); - if (hosts == null || hosts.isEmpty()) { - continue; - } - - for (HostVO h : hosts) { - if (h.getStatus() == Status.Up) { - s_logger.debug("Pick up host that has hypervisor type " + h.getHypervisorType() + " in cluster " + - cv.getId() + " to start domain router for OVM"); - return h.getHypervisorType(); - } - } - } - - String errMsg = "Cannot find an available cluster in Pod " - + podId - + " to start domain router for Ovm. \n Ovm won't support any system vm including domain router, " + - "please make sure you have a cluster with hypervisor type of any of xenserver/KVM/Vmware in the same pod" + - " with Ovm cluster. And there is at least one host in UP status in that cluster."; - throw new CloudRuntimeException(errMsg); - } - - @DB - protected List findOrDeployVirtualRouterInGuestNetwork(Network guestNetwork, DeployDestination dest, Account owner, - boolean isRedundant, Map params) throws ConcurrentOperationException, - InsufficientCapacityException, ResourceUnavailableException { - - assert guestNetwork.getState() == Network.State.Implemented || guestNetwork.getState() == Network.State.Setup || - guestNetwork.getState() == Network.State.Implementing : "Network is not yet fully implemented: " - + guestNetwork; - assert guestNetwork.getTrafficType() == TrafficType.Guest; - - - - //1) Get deployment plan and find out the list of routers - boolean isPodBased = (dest.getDataCenter().getNetworkType() == NetworkType.Basic || - _networkMgr.areServicesSupportedInNetwork(guestNetwork.getId(), Service.SecurityGroup)) - && guestNetwork.getTrafficType() == TrafficType.Guest; - - Pair> planAndRouters = getDeploymentPlanAndRouters(isPodBased, dest, guestNetwork.getId()); - DeploymentPlan plan = planAndRouters.first(); - List routers = planAndRouters.second(); - - //2) Figure out required routers count - int routerCount = 1; - if (isRedundant) { - routerCount = 2; - } - - /* If it is the single router network, then keep it untouched */ - for (DomainRouterVO router : routers) { - if (!router.getIsRedundantRouter() || isPodBased) { - routerCount = 1; - break; - } - } - - /* If old network is redundant but new is single router, then routers.size() = 2 but routerCount = 1 */ - if (routers.size() >= routerCount) { - return routers; - } - - if (routers.size() >= 5) { - s_logger.error("Too much redundant routers!"); - } - - Network network = _networkDao.acquireInLockTable(guestNetwork.getId()); - if (network == null) { - throw new ConcurrentOperationException("Unable to lock network " + guestNetwork.getId()); - } - - try { - //Check if providers are supported in the physical networks - VirtualRouterProviderType type = VirtualRouterProviderType.VirtualRouter; - Long physicalNetworkId = _networkMgr.getPhysicalNetworkId(network); - PhysicalNetworkServiceProvider provider = _physicalProviderDao.findByServiceProvider(physicalNetworkId, type.toString()); - if (provider == null) { - throw new CloudRuntimeException("Cannot find service provider " + type.toString() + " in physical network " + physicalNetworkId); - } - VirtualRouterProvider vrProvider = _vrProviderDao.findByNspIdAndType(provider.getId(), type); - if (vrProvider == null) { - throw new CloudRuntimeException("Cannot find virtual router provider " + type.toString()+ " as service provider " + provider.getId()); - } - - if (_networkMgr.isNetworkSystem(guestNetwork) || guestNetwork.getGuestType() == Network.GuestType.Shared) { - owner = _accountMgr.getAccount(Account.ACCOUNT_ID_SYSTEM); - } - - //Check if public network has to be set on VR - boolean publicNetwork = false; - if (_networkMgr.isProviderSupportServiceInNetwork(guestNetwork.getId(), Service.SourceNat, Provider.VirtualRouter)) { - publicNetwork = true; - } - if (isRedundant && !publicNetwork) { - s_logger.error("Didn't support redundant virtual router without public network!"); - return null; - } - - Long offeringId = _networkOfferingDao.findById(guestNetwork.getNetworkOfferingId()).getServiceOfferingId(); - if (offeringId == null) { - offeringId = _offering.getId(); - } - - PublicIp sourceNatIp = null; - if (publicNetwork) { - sourceNatIp = _networkMgr.assignSourceNatIpAddressToGuestNetwork(owner, guestNetwork); - } - - //3) deploy virtual router(s) - int count = routerCount - routers.size(); - for (int i = 0; i < count; i++) { - List> networks = createRouterNetworks(owner, isRedundant, plan, guestNetwork, - new Pair(publicNetwork, sourceNatIp)); - DomainRouterVO router = deployRouter(owner, dest, plan, params, isRedundant, vrProvider, offeringId, - null, networks); - - _routerDao.addRouterToGuestNetwork(router, network); - - routers.add(router); - } - } finally { - if (network != null) { - _networkDao.releaseFromLockTable(network.getId()); - } - } - return routers; - } - - protected DomainRouterVO deployRouter(Account owner, DeployDestination dest, DeploymentPlan plan, Map params, - boolean isRedundant, VirtualRouterProvider vrProvider, long svcOffId, - Long vpcId, List> networks) throws ConcurrentOperationException, - InsufficientAddressCapacityException, InsufficientServerCapacityException, InsufficientCapacityException, - StorageUnavailableException, ResourceUnavailableException { - - long id = _routerDao.getNextInSequence(Long.class, "id"); - if (s_logger.isDebugEnabled()) { - s_logger.debug("Creating the router " + id + " in datacenter " + dest.getDataCenter()); - } - - ServiceOfferingVO routerOffering = _serviceOfferingDao.findById(svcOffId); - - // Router is the network element, we don't know the hypervisor type yet. - //Try to allocate the domR twice using diff hypervisors, and when failed both times, throw the exception up - List supportedHypervisors = new ArrayList(); - HypervisorType defaults = _resourceMgr.getDefaultHypervisor(dest.getDataCenter().getId()); - if (defaults != HypervisorType.None) { - supportedHypervisors.add(defaults); - } - - if (dest.getCluster() != null) { - if (dest.getCluster().getHypervisorType() == HypervisorType.Ovm) { - supportedHypervisors.add(getClusterToStartDomainRouterForOvm(dest.getCluster().getPodId())); - } else { - supportedHypervisors.add(dest.getCluster().getHypervisorType()); - } - } else { - supportedHypervisors = _resourceMgr.getSupportedHypervisorTypes(dest.getDataCenter().getId(), true, - plan.getPodId()); - } - - if (supportedHypervisors.isEmpty()) { - if (plan.getPodId() != null) { - throw new InsufficientServerCapacityException("Unable to create virtual router, " + - "there are no clusters in the pod ", Pod.class, plan.getPodId()); - } - throw new InsufficientServerCapacityException("Unable to create virtual router, " + - "there are no clusters in the zone ", DataCenter.class, dest.getDataCenter().getId()); - } - - int allocateRetry = 0; - int startRetry = 0; - DomainRouterVO router = null; - for (Iterator iter = supportedHypervisors.iterator();iter.hasNext();) { - HypervisorType hType = iter.next(); - try { - s_logger.debug("Allocating the domR with the hypervisor type " + hType); - VMTemplateVO template = _templateDao.findRoutingTemplate(hType); - - if (template == null) { - s_logger.debug(hType + " won't support system vm, skip it"); - continue; - } - - boolean offerHA = routerOffering.getOfferHA(); - /* We don't provide HA to redundant router VMs, admin should own it all, and redundant router themselves are HA */ - if (isRedundant) { - offerHA = false; - } - - router = new DomainRouterVO(id, routerOffering.getId(), vrProvider.getId(), - VirtualMachineName.getRouterName(id, _instance), template.getId(), template.getHypervisorType(), - template.getGuestOSId(), owner.getDomainId(), owner.getId(), isRedundant, 0, false, - RedundantState.UNKNOWN, offerHA, false, vpcId); - router.setRole(Role.VIRTUAL_ROUTER); - router = _itMgr.allocate(router, template, routerOffering, networks, plan, null, owner); - } catch (InsufficientCapacityException ex) { - if (allocateRetry < 2 && iter.hasNext()) { - s_logger.debug("Failed to allocate the domR with hypervisor type " + hType + ", retrying one more time"); - continue; - } else { - throw ex; - } - } finally { - allocateRetry++; - } - - try { - router = startVirtualRouter(router, _accountMgr.getSystemUser(), _accountMgr.getSystemAccount(), params); - break; - } catch (InsufficientCapacityException ex) { - if (startRetry < 2 && iter.hasNext()) { - s_logger.debug("Failed to start the domR " + router + " with hypervisor type " + hType + ", " + - "destroying it and recreating one more time"); - //destroy the router - destroyRouter(router.getId()); - continue; - } else { - throw ex; - } - } finally { - startRetry++; - } - } - - return router; - } - - protected List> createRouterNetworks(Account owner, boolean isRedundant, - DeploymentPlan plan, Network guestNetwork, Pair publicNetwork) throws ConcurrentOperationException, - InsufficientAddressCapacityException { - - - boolean setupPublicNetwork = false; - if (publicNetwork != null) { - setupPublicNetwork = publicNetwork.first(); - } - - //Form networks - List> networks = new ArrayList>(3); - - //1) Guest network - boolean hasGuestNetwork = false; - if (guestNetwork != null) { - s_logger.debug("Adding nic for Virtual Router in Guest network " + guestNetwork); - String defaultNetworkStartIp = null; - if (guestNetwork.getCidr() != null && !setupPublicNetwork) { - String startIp = _networkMgr.getStartIpAddress(guestNetwork.getId()); - if (startIp != null && _ipAddressDao.findByIpAndSourceNetworkId(guestNetwork.getId(), startIp).getAllocatedTime() == null) { - defaultNetworkStartIp = startIp; - } else if (s_logger.isDebugEnabled()){ - s_logger.debug("First ip " + startIp + " in network id=" + guestNetwork.getId() + - " is already allocated, can't use it for domain router; will get random ip address from the range"); - } - } - - NicProfile gatewayNic = new NicProfile(defaultNetworkStartIp); - if (setupPublicNetwork) { - if (isRedundant) { - gatewayNic.setIp4Address(_networkMgr.acquireGuestIpAddress(guestNetwork, null)); - } else { - gatewayNic.setIp4Address(guestNetwork.getGateway()); - } - gatewayNic.setBroadcastUri(guestNetwork.getBroadcastUri()); - gatewayNic.setBroadcastType(guestNetwork.getBroadcastDomainType()); - gatewayNic.setIsolationUri(guestNetwork.getBroadcastUri()); - gatewayNic.setMode(guestNetwork.getMode()); - String gatewayCidr = guestNetwork.getCidr(); - gatewayNic.setNetmask(NetUtils.getCidrNetmask(gatewayCidr)); - } else { - gatewayNic.setDefaultNic(true); - } - - networks.add(new Pair((NetworkVO) guestNetwork, gatewayNic)); - hasGuestNetwork = true; - } - - //2) Control network - s_logger.debug("Adding nic for Virtual Router in Control network "); - List offerings = _networkMgr.getSystemAccountNetworkOfferings(NetworkOfferingVO.SystemControlNetwork); - NetworkOfferingVO controlOffering = offerings.get(0); - NetworkVO controlConfig = _networkMgr.setupNetwork(_systemAcct, controlOffering, plan, null, null, false).get(0); - networks.add(new Pair(controlConfig, null)); - - - //3) Public network - if (setupPublicNetwork) { - PublicIp sourceNatIp = publicNetwork.second(); - s_logger.debug("Adding nic for Virtual Router in Public network "); - //if source nat service is supported by the network, get the source nat ip address - NicProfile defaultNic = new NicProfile(); - defaultNic.setDefaultNic(true); - defaultNic.setIp4Address(sourceNatIp.getAddress().addr()); - defaultNic.setGateway(sourceNatIp.getGateway()); - defaultNic.setNetmask(sourceNatIp.getNetmask()); - defaultNic.setMacAddress(sourceNatIp.getMacAddress()); - defaultNic.setBroadcastType(BroadcastDomainType.Vlan); - defaultNic.setBroadcastUri(BroadcastDomainType.Vlan.toUri(sourceNatIp.getVlanTag())); - defaultNic.setIsolationUri(IsolationType.Vlan.toUri(sourceNatIp.getVlanTag())); - if (hasGuestNetwork) { - defaultNic.setDeviceId(2); - } - NetworkOfferingVO publicOffering = _networkMgr.getSystemAccountNetworkOfferings(NetworkOfferingVO.SystemPublicNetwork).get(0); - List publicNetworks = _networkMgr.setupNetwork(_systemAcct, publicOffering, plan, null, null, false); - networks.add(new Pair(publicNetworks.get(0), defaultNic)); - } - - return networks; - } - - - protected Pair> getDeploymentPlanAndRouters(boolean isPodBased, - DeployDestination dest, long guestNetworkId) { - long dcId = dest.getDataCenter().getId(); - List routers = null; - DeploymentPlan plan = new DataCenterDeployment(dcId); - if (isPodBased) { - Pod pod = dest.getPod(); - Long podId = null; - if (pod != null) { - podId = pod.getId(); - } else { - throw new CloudRuntimeException("Pod id is expected in deployment destination"); - } - routers = _routerDao.listByNetworkAndPodAndRole(guestNetworkId, podId, Role.VIRTUAL_ROUTER); - plan = new DataCenterDeployment(dcId, podId, null, null, null, null); - } else { - routers = _routerDao.listByNetworkAndRole(guestNetworkId, Role.VIRTUAL_ROUTER); - } - - return new Pair>(plan, routers); - } - - - private DomainRouterVO startVirtualRouter(DomainRouterVO router, User user, Account caller, Map params) - throws StorageUnavailableException, InsufficientCapacityException, - ConcurrentOperationException, ResourceUnavailableException { - - if (router.getRole() != Role.VIRTUAL_ROUTER || !router.getIsRedundantRouter()) { - return this.start(router, user, caller, params, null); - } - - if (router.getState() == State.Running) { - s_logger.debug("Redundant router " + router.getInstanceName() + " is already running!"); - return router; - } - - DataCenterDeployment plan = new DataCenterDeployment(0, null, null, null, null, null); - DomainRouterVO result = null; - assert router.getIsRedundantRouter(); + if (conn.getState() != Site2SiteVpnConnection.State.Connected && + conn.getState() != Site2SiteVpnConnection.State.Disconnected) { + continue; + } + Site2SiteVpnConnection.State oldState = conn.getState(); + Site2SiteCustomerGateway gw = _s2sCustomerGatewayDao.findById(conn.getCustomerGatewayId()); + if (answer.isConnected(gw.getGatewayIp())) { + conn.setState(Site2SiteVpnConnection.State.Connected); + } else { + conn.setState(Site2SiteVpnConnection.State.Disconnected); + } + _s2sVpnConnectionDao.persist(conn); + if (oldState != conn.getState()) { + String title = "Site-to-site Vpn Connection to " + gw.getName() + + " just switch from " + oldState + " to " + conn.getState(); + String context = "Site-to-site Vpn Connection to " + gw.getName() + " on router " + router.getHostName() + + "(id: " + router.getId() + ") " + " just switch from " + oldState + " to " + conn.getState(); + s_logger.info(context); + _alertMgr.sendAlert(AlertManager.ALERT_TYPE_DOMAIN_ROUTER, + router.getDataCenterIdToDeployIn(), router.getPodIdToDeployIn(), title, context); + } + } finally { + _s2sVpnConnectionDao.releaseFromLockTable(lock.getId()); + } + } + } + } + } + + protected void updateRoutersRedundantState(List routers) { + boolean updated = false; + for (DomainRouterVO router : routers) { + updated = false; + if (!router.getIsRedundantRouter()) { + continue; + } + RedundantState prevState = router.getRedundantState(); + if (router.getState() != State.Running) { + router.setRedundantState(RedundantState.UNKNOWN); + router.setIsPriorityBumpUp(false); + updated = true; + } else { + String privateIP = router.getPrivateIpAddress(); + HostVO host = _hostDao.findById(router.getHostId()); + if (host == null || host.getStatus() != Status.Up) { + router.setRedundantState(RedundantState.UNKNOWN); + updated = true; + } else if (host.getManagementServerId() != ManagementServerNode.getManagementServerId()) { + /* Only cover hosts managed by this management server */ + continue; + } else if (privateIP != null) { + final CheckRouterCommand command = new CheckRouterCommand(); + command.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); + command.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); + command.setWait(60); + final Answer origAnswer = _agentMgr.easySend(router.getHostId(), command); + CheckRouterAnswer answer = null; + if (origAnswer instanceof CheckRouterAnswer) { + answer = (CheckRouterAnswer)origAnswer; + } else { + s_logger.warn("Unable to update router " + router.getHostName() + "'s status"); + } + RedundantState state = RedundantState.UNKNOWN; + boolean isBumped = router.getIsPriorityBumpUp(); + if (answer != null && answer.getResult()) { + state = answer.getState(); + isBumped = answer.isBumped(); + } + router.setRedundantState(state); + router.setIsPriorityBumpUp(isBumped); + updated = true; + } + } + if (updated) { + Transaction txn = Transaction.open(Transaction.CLOUD_DB); + try { + txn.start(); + _routerDao.update(router.getId(), router); + txn.commit(); + } catch (Exception e) { + txn.rollback(); + s_logger.warn("Unable to update router status for account: " + router.getAccountId()); + } finally { + txn.close(); + } + } + RedundantState currState = router.getRedundantState(); + if (prevState != currState) { + String title = "Redundant virtual router " + router.getInstanceName() + + " just switch from " + prevState + " to " + currState; + String context = "Redundant virtual router (name: " + router.getHostName() + ", id: " + router.getId() + ") " + + " just switch from " + prevState + " to " + currState; + s_logger.info(context); + if (currState == RedundantState.MASTER) { + _alertMgr.sendAlert(AlertManager.ALERT_TYPE_DOMAIN_ROUTER, + router.getDataCenterIdToDeployIn(), router.getPodIdToDeployIn(), title, context); + } + } + } + } + + //Ensure router status is update to date before execute this function. The function would try best to recover all routers except MASTER + protected void recoverRedundantNetwork(DomainRouterVO masterRouter, DomainRouterVO backupRouter) { + UserContext context = UserContext.current(); + context.setAccountId(1); + if (masterRouter.getState() == State.Running && backupRouter.getState() == State.Running) { + HostVO masterHost = _hostDao.findById(masterRouter.getHostId()); + HostVO backupHost = _hostDao.findById(backupRouter.getHostId()); + if (masterHost.getStatus() == Status.Up && backupHost.getStatus() == Status.Up) { + String title = "Reboot " + backupRouter.getInstanceName() + " to ensure redundant virtual routers work"; + if (s_logger.isDebugEnabled()) { + s_logger.debug(title); + } + _alertMgr.sendAlert(AlertManager.ALERT_TYPE_DOMAIN_ROUTER, + backupRouter.getDataCenterIdToDeployIn(), backupRouter.getPodIdToDeployIn(), title, title); + try { + rebootRouter(backupRouter.getId(), false); + } catch (ConcurrentOperationException e) { + s_logger.warn("Fail to reboot " + backupRouter.getInstanceName(), e); + } catch (ResourceUnavailableException e) { + s_logger.warn("Fail to reboot " + backupRouter.getInstanceName(), e); + } catch (InsufficientCapacityException e) { + s_logger.warn("Fail to reboot " + backupRouter.getInstanceName(), e); + } + } + } + } + + private int getRealPriority(DomainRouterVO router) { + int priority = router.getPriority(); + if (router.getIsPriorityBumpUp()) { + priority += DEFAULT_DELTA; + } + return priority; + } + + protected class CheckRouterTask implements Runnable { + + public CheckRouterTask() { + } + + /* + * In order to make fail-over works well at any time, we have to ensure: + * 1. Backup router's priority = Master's priority - DELTA + 1 + * 2. Backup router's priority hasn't been bumped up. + */ + private void checkSanity(List routers) { + Set checkedNetwork = new HashSet(); + for (DomainRouterVO router : routers) { + if (!router.getIsRedundantRouter()) { + continue; + } + + List routerGuestNtwkIds = _routerDao.getRouterNetworks(router.getId()); + + for (Long routerGuestNtwkId : routerGuestNtwkIds) { + if (checkedNetwork.contains(routerGuestNtwkId)) { + continue; + } + checkedNetwork.add(routerGuestNtwkId); + List checkingRouters = _routerDao.listByNetworkAndRole(routerGuestNtwkId, Role.VIRTUAL_ROUTER); + if (checkingRouters.size() != 2) { + continue; + } + DomainRouterVO masterRouter = null; + DomainRouterVO backupRouter = null; + for (DomainRouterVO r : checkingRouters) { + if (r.getRedundantState() == RedundantState.MASTER) { + if (masterRouter == null) { + masterRouter = r; + } else { + //Duplicate master! We give up, until the admin fix duplicate MASTER issue + break; + } + } else if (r.getRedundantState() == RedundantState.BACKUP) { + if (backupRouter == null) { + backupRouter = r; + } else { + break; + } + } + } + if (masterRouter != null && backupRouter != null) { + if (getRealPriority(masterRouter) - DEFAULT_DELTA + 1 != getRealPriority(backupRouter) || backupRouter.getIsPriorityBumpUp()) { + recoverRedundantNetwork(masterRouter, backupRouter); + } + } + } + } + } + + private void checkDuplicateMaster(List routers) { + Map networkRouterMaps = new HashMap(); + for (DomainRouterVO router : routers) { + List routerGuestNtwkIds = _routerDao.getRouterNetworks(router.getId()); + + for (Long routerGuestNtwkId : routerGuestNtwkIds) { + if (router.getRedundantState() == RedundantState.MASTER) { + if (networkRouterMaps.containsKey(routerGuestNtwkId)) { + DomainRouterVO dupRouter = networkRouterMaps.get(routerGuestNtwkId); + String title = "More than one redundant virtual router is in MASTER state! Router " + router.getHostName() + " and router " + dupRouter.getHostName(); + String context = "Virtual router (name: " + router.getHostName() + ", id: " + router.getId() + " and router (name: " + + dupRouter.getHostName() + ", id: " + router.getId() + ") are both in MASTER state! If the problem persist, restart both of routers. "; + + _alertMgr.sendAlert(AlertManager.ALERT_TYPE_DOMAIN_ROUTER, router.getDataCenterIdToDeployIn(), router.getPodIdToDeployIn(), title, context); + _alertMgr.sendAlert(AlertManager.ALERT_TYPE_DOMAIN_ROUTER, dupRouter.getDataCenterIdToDeployIn(), dupRouter.getPodIdToDeployIn(), title, context); + } else { + networkRouterMaps.put(routerGuestNtwkId, router); + } + } + } + } + } + + @Override + public void run() { + try { + final List routers = _routerDao.listIsolatedByHostId(null); + s_logger.debug("Found " + routers.size() + " routers. "); + + updateRoutersRedundantState(routers); + updateSite2SiteVpnConnectionState(routers); + + /* FIXME assumed the a pair of redundant routers managed by same mgmt server, + * then the update above can get the latest status */ + checkDuplicateMaster(routers); + checkSanity(routers); + } catch (Exception ex) { + s_logger.error("Fail to complete the CheckRouterTask! ", ex); + } + } + } + + + private final int DEFAULT_PRIORITY = 100; + private final int DEFAULT_DELTA = 2; + + protected int getUpdatedPriority(Network guestNetwork, List routers, DomainRouterVO exclude) throws InsufficientVirtualNetworkCapcityException { + int priority; + if (routers.size() == 0) { + priority = DEFAULT_PRIORITY; + } else { + int maxPriority = 0; + for (DomainRouterVO r : routers) { + if (!r.getIsRedundantRouter()) { + throw new CloudRuntimeException("Redundant router is mixed with single router in one network!"); + } + //FIXME Assume the maxPriority one should be running or just created. + if (r.getId() != exclude.getId() && getRealPriority(r) > maxPriority) { + maxPriority = getRealPriority(r); + } + } + if (maxPriority == 0) { + return DEFAULT_PRIORITY; + } + if (maxPriority < 20) { + s_logger.error("Current maximum priority is too low!"); + throw new InsufficientVirtualNetworkCapcityException("Current maximum priority is too low as " + maxPriority + "!", + guestNetwork.getId()); + } else if (maxPriority > 200) { + s_logger.error("Too many times fail-over happened! Current maximum priority is too high as " + maxPriority + "!"); + throw new InsufficientVirtualNetworkCapcityException("Too many times fail-over happened! Current maximum priority is too high as " + + maxPriority + "!", guestNetwork.getId()); + } + priority = maxPriority - DEFAULT_DELTA + 1; + } + return priority; + } + + /* + * Ovm won't support any system. So we have to choose a partner cluster in the same pod to start domain router for us + */ + private HypervisorType getClusterToStartDomainRouterForOvm(long podId) { + List clusters = _clusterDao.listByPodId(podId); + for (ClusterVO cv : clusters) { + if (cv.getHypervisorType() == HypervisorType.Ovm || cv.getHypervisorType() == HypervisorType.BareMetal) { + continue; + } + + List hosts = _resourceMgr.listAllHostsInCluster(cv.getId()); + if (hosts == null || hosts.isEmpty()) { + continue; + } + + for (HostVO h : hosts) { + if (h.getStatus() == Status.Up) { + s_logger.debug("Pick up host that has hypervisor type " + h.getHypervisorType() + " in cluster " + + cv.getId() + " to start domain router for OVM"); + return h.getHypervisorType(); + } + } + } + + String errMsg = "Cannot find an available cluster in Pod " + + podId + + " to start domain router for Ovm. \n Ovm won't support any system vm including domain router, " + + "please make sure you have a cluster with hypervisor type of any of xenserver/KVM/Vmware in the same pod" + + " with Ovm cluster. And there is at least one host in UP status in that cluster."; + throw new CloudRuntimeException(errMsg); + } + + @DB + protected List findOrDeployVirtualRouterInGuestNetwork(Network guestNetwork, DeployDestination dest, Account owner, + boolean isRedundant, Map params) throws ConcurrentOperationException, + InsufficientCapacityException, ResourceUnavailableException { + + assert guestNetwork.getState() == Network.State.Implemented || guestNetwork.getState() == Network.State.Setup || + guestNetwork.getState() == Network.State.Implementing : "Network is not yet fully implemented: " + + guestNetwork; + assert guestNetwork.getTrafficType() == TrafficType.Guest; + + + + //1) Get deployment plan and find out the list of routers + boolean isPodBased = (dest.getDataCenter().getNetworkType() == NetworkType.Basic || + _networkMgr.areServicesSupportedInNetwork(guestNetwork.getId(), Service.SecurityGroup)) + && guestNetwork.getTrafficType() == TrafficType.Guest; + + Pair> planAndRouters = getDeploymentPlanAndRouters(isPodBased, dest, guestNetwork.getId()); + DeploymentPlan plan = planAndRouters.first(); + List routers = planAndRouters.second(); + + //2) Figure out required routers count + int routerCount = 1; + if (isRedundant) { + routerCount = 2; + } + + /* If it is the single router network, then keep it untouched */ + for (DomainRouterVO router : routers) { + if (!router.getIsRedundantRouter() || isPodBased) { + routerCount = 1; + break; + } + } + + /* If old network is redundant but new is single router, then routers.size() = 2 but routerCount = 1 */ + if (routers.size() >= routerCount) { + return routers; + } + + if (routers.size() >= 5) { + s_logger.error("Too much redundant routers!"); + } + + Network network = _networkDao.acquireInLockTable(guestNetwork.getId()); + if (network == null) { + throw new ConcurrentOperationException("Unable to lock network " + guestNetwork.getId()); + } + + try { + //Check if providers are supported in the physical networks + VirtualRouterProviderType type = VirtualRouterProviderType.VirtualRouter; + Long physicalNetworkId = _networkMgr.getPhysicalNetworkId(network); + PhysicalNetworkServiceProvider provider = _physicalProviderDao.findByServiceProvider(physicalNetworkId, type.toString()); + if (provider == null) { + throw new CloudRuntimeException("Cannot find service provider " + type.toString() + " in physical network " + physicalNetworkId); + } + VirtualRouterProvider vrProvider = _vrProviderDao.findByNspIdAndType(provider.getId(), type); + if (vrProvider == null) { + throw new CloudRuntimeException("Cannot find virtual router provider " + type.toString()+ " as service provider " + provider.getId()); + } + + if (_networkMgr.isNetworkSystem(guestNetwork) || guestNetwork.getGuestType() == Network.GuestType.Shared) { + owner = _accountMgr.getAccount(Account.ACCOUNT_ID_SYSTEM); + } + + //Check if public network has to be set on VR + boolean publicNetwork = false; + if (_networkMgr.isProviderSupportServiceInNetwork(guestNetwork.getId(), Service.SourceNat, Provider.VirtualRouter)) { + publicNetwork = true; + } + if (isRedundant && !publicNetwork) { + s_logger.error("Didn't support redundant virtual router without public network!"); + return null; + } + + Long offeringId = _networkOfferingDao.findById(guestNetwork.getNetworkOfferingId()).getServiceOfferingId(); + if (offeringId == null) { + offeringId = _offering.getId(); + } + + PublicIp sourceNatIp = null; + if (publicNetwork) { + sourceNatIp = _networkMgr.assignSourceNatIpAddressToGuestNetwork(owner, guestNetwork); + } + + //3) deploy virtual router(s) + int count = routerCount - routers.size(); + for (int i = 0; i < count; i++) { + List> networks = createRouterNetworks(owner, isRedundant, plan, guestNetwork, + new Pair(publicNetwork, sourceNatIp)); + DomainRouterVO router = deployRouter(owner, dest, plan, params, isRedundant, vrProvider, offeringId, + null, networks); + + _routerDao.addRouterToGuestNetwork(router, network); + + routers.add(router); + } + } finally { + if (network != null) { + _networkDao.releaseFromLockTable(network.getId()); + } + } + return routers; + } + + protected DomainRouterVO deployRouter(Account owner, DeployDestination dest, DeploymentPlan plan, Map params, + boolean isRedundant, VirtualRouterProvider vrProvider, long svcOffId, + Long vpcId, List> networks) throws ConcurrentOperationException, + InsufficientAddressCapacityException, InsufficientServerCapacityException, InsufficientCapacityException, + StorageUnavailableException, ResourceUnavailableException { + + long id = _routerDao.getNextInSequence(Long.class, "id"); + if (s_logger.isDebugEnabled()) { + s_logger.debug("Creating the router " + id + " in datacenter " + dest.getDataCenter()); + } + + ServiceOfferingVO routerOffering = _serviceOfferingDao.findById(svcOffId); + + // Router is the network element, we don't know the hypervisor type yet. + //Try to allocate the domR twice using diff hypervisors, and when failed both times, throw the exception up + List supportedHypervisors = new ArrayList(); + HypervisorType defaults = _resourceMgr.getDefaultHypervisor(dest.getDataCenter().getId()); + if (defaults != HypervisorType.None) { + supportedHypervisors.add(defaults); + } + + if (dest.getCluster() != null) { + if (dest.getCluster().getHypervisorType() == HypervisorType.Ovm) { + supportedHypervisors.add(getClusterToStartDomainRouterForOvm(dest.getCluster().getPodId())); + } else { + supportedHypervisors.add(dest.getCluster().getHypervisorType()); + } + } else { + supportedHypervisors = _resourceMgr.getSupportedHypervisorTypes(dest.getDataCenter().getId(), true, + plan.getPodId()); + } + + if (supportedHypervisors.isEmpty()) { + if (plan.getPodId() != null) { + throw new InsufficientServerCapacityException("Unable to create virtual router, " + + "there are no clusters in the pod ", Pod.class, plan.getPodId()); + } + throw new InsufficientServerCapacityException("Unable to create virtual router, " + + "there are no clusters in the zone ", DataCenter.class, dest.getDataCenter().getId()); + } + + int allocateRetry = 0; + int startRetry = 0; + DomainRouterVO router = null; + for (Iterator iter = supportedHypervisors.iterator();iter.hasNext();) { + HypervisorType hType = iter.next(); + try { + s_logger.debug("Allocating the domR with the hypervisor type " + hType); + VMTemplateVO template = _templateDao.findRoutingTemplate(hType); + + if (template == null) { + s_logger.debug(hType + " won't support system vm, skip it"); + continue; + } + + boolean offerHA = routerOffering.getOfferHA(); + /* We don't provide HA to redundant router VMs, admin should own it all, and redundant router themselves are HA */ + if (isRedundant) { + offerHA = false; + } + + router = new DomainRouterVO(id, routerOffering.getId(), vrProvider.getId(), + VirtualMachineName.getRouterName(id, _instance), template.getId(), template.getHypervisorType(), + template.getGuestOSId(), owner.getDomainId(), owner.getId(), isRedundant, 0, false, + RedundantState.UNKNOWN, offerHA, false, vpcId); + router.setRole(Role.VIRTUAL_ROUTER); + router = _itMgr.allocate(router, template, routerOffering, networks, plan, null, owner); + } catch (InsufficientCapacityException ex) { + if (allocateRetry < 2 && iter.hasNext()) { + s_logger.debug("Failed to allocate the domR with hypervisor type " + hType + ", retrying one more time"); + continue; + } else { + throw ex; + } + } finally { + allocateRetry++; + } + + try { + router = startVirtualRouter(router, _accountMgr.getSystemUser(), _accountMgr.getSystemAccount(), params); + break; + } catch (InsufficientCapacityException ex) { + if (startRetry < 2 && iter.hasNext()) { + s_logger.debug("Failed to start the domR " + router + " with hypervisor type " + hType + ", " + + "destroying it and recreating one more time"); + //destroy the router + destroyRouter(router.getId()); + continue; + } else { + throw ex; + } + } finally { + startRetry++; + } + } + + return router; + } + + protected List> createRouterNetworks(Account owner, boolean isRedundant, + DeploymentPlan plan, Network guestNetwork, Pair publicNetwork) throws ConcurrentOperationException, + InsufficientAddressCapacityException { + + + boolean setupPublicNetwork = false; + if (publicNetwork != null) { + setupPublicNetwork = publicNetwork.first(); + } + + //Form networks + List> networks = new ArrayList>(3); + + //1) Guest network + boolean hasGuestNetwork = false; + if (guestNetwork != null) { + s_logger.debug("Adding nic for Virtual Router in Guest network " + guestNetwork); + String defaultNetworkStartIp = null; + if (guestNetwork.getCidr() != null && !setupPublicNetwork) { + String startIp = _networkMgr.getStartIpAddress(guestNetwork.getId()); + if (startIp != null && _ipAddressDao.findByIpAndSourceNetworkId(guestNetwork.getId(), startIp).getAllocatedTime() == null) { + defaultNetworkStartIp = startIp; + } else if (s_logger.isDebugEnabled()){ + s_logger.debug("First ip " + startIp + " in network id=" + guestNetwork.getId() + + " is already allocated, can't use it for domain router; will get random ip address from the range"); + } + } + + NicProfile gatewayNic = new NicProfile(defaultNetworkStartIp); + if (setupPublicNetwork) { + if (isRedundant) { + gatewayNic.setIp4Address(_networkMgr.acquireGuestIpAddress(guestNetwork, null)); + } else { + gatewayNic.setIp4Address(guestNetwork.getGateway()); + } + gatewayNic.setBroadcastUri(guestNetwork.getBroadcastUri()); + gatewayNic.setBroadcastType(guestNetwork.getBroadcastDomainType()); + gatewayNic.setIsolationUri(guestNetwork.getBroadcastUri()); + gatewayNic.setMode(guestNetwork.getMode()); + String gatewayCidr = guestNetwork.getCidr(); + gatewayNic.setNetmask(NetUtils.getCidrNetmask(gatewayCidr)); + } else { + gatewayNic.setDefaultNic(true); + } + + networks.add(new Pair((NetworkVO) guestNetwork, gatewayNic)); + hasGuestNetwork = true; + } + + //2) Control network + s_logger.debug("Adding nic for Virtual Router in Control network "); + List offerings = _networkMgr.getSystemAccountNetworkOfferings(NetworkOfferingVO.SystemControlNetwork); + NetworkOfferingVO controlOffering = offerings.get(0); + NetworkVO controlConfig = _networkMgr.setupNetwork(_systemAcct, controlOffering, plan, null, null, false).get(0); + networks.add(new Pair(controlConfig, null)); + + + //3) Public network + if (setupPublicNetwork) { + PublicIp sourceNatIp = publicNetwork.second(); + s_logger.debug("Adding nic for Virtual Router in Public network "); + //if source nat service is supported by the network, get the source nat ip address + NicProfile defaultNic = new NicProfile(); + defaultNic.setDefaultNic(true); + defaultNic.setIp4Address(sourceNatIp.getAddress().addr()); + defaultNic.setGateway(sourceNatIp.getGateway()); + defaultNic.setNetmask(sourceNatIp.getNetmask()); + defaultNic.setMacAddress(sourceNatIp.getMacAddress()); + defaultNic.setBroadcastType(BroadcastDomainType.Vlan); + defaultNic.setBroadcastUri(BroadcastDomainType.Vlan.toUri(sourceNatIp.getVlanTag())); + defaultNic.setIsolationUri(IsolationType.Vlan.toUri(sourceNatIp.getVlanTag())); + if (hasGuestNetwork) { + defaultNic.setDeviceId(2); + } + NetworkOfferingVO publicOffering = _networkMgr.getSystemAccountNetworkOfferings(NetworkOfferingVO.SystemPublicNetwork).get(0); + List publicNetworks = _networkMgr.setupNetwork(_systemAcct, publicOffering, plan, null, null, false); + networks.add(new Pair(publicNetworks.get(0), defaultNic)); + } + + return networks; + } + + + protected Pair> getDeploymentPlanAndRouters(boolean isPodBased, + DeployDestination dest, long guestNetworkId) { + long dcId = dest.getDataCenter().getId(); + List routers = null; + DeploymentPlan plan = new DataCenterDeployment(dcId); + if (isPodBased) { + Pod pod = dest.getPod(); + Long podId = null; + if (pod != null) { + podId = pod.getId(); + } else { + throw new CloudRuntimeException("Pod id is expected in deployment destination"); + } + routers = _routerDao.listByNetworkAndPodAndRole(guestNetworkId, podId, Role.VIRTUAL_ROUTER); + plan = new DataCenterDeployment(dcId, podId, null, null, null, null); + } else { + routers = _routerDao.listByNetworkAndRole(guestNetworkId, Role.VIRTUAL_ROUTER); + } + + return new Pair>(plan, routers); + } + + + private DomainRouterVO startVirtualRouter(DomainRouterVO router, User user, Account caller, Map params) + throws StorageUnavailableException, InsufficientCapacityException, + ConcurrentOperationException, ResourceUnavailableException { + + if (router.getRole() != Role.VIRTUAL_ROUTER || !router.getIsRedundantRouter()) { + return this.start(router, user, caller, params, null); + } + + if (router.getState() == State.Running) { + s_logger.debug("Redundant router " + router.getInstanceName() + " is already running!"); + return router; + } + + DataCenterDeployment plan = new DataCenterDeployment(0, null, null, null, null, null); + DomainRouterVO result = null; + assert router.getIsRedundantRouter(); List networkIds = _routerDao.getRouterNetworks(router.getId()); //Not support VPC now if (networkIds.size() > 1) { throw new ResourceUnavailableException("Unable to support more than one guest network for redundant router now!", DataCenter.class, router.getDataCenterIdToDeployIn()); } - DomainRouterVO routerToBeAvoid = null; + DomainRouterVO routerToBeAvoid = null; if (networkIds.size() != 0) { List routerList = _routerDao.findByNetwork(networkIds.get(0)); for (DomainRouterVO rrouter : routerList) { @@ -1609,1781 +1609,1781 @@ public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplian routerToBeAvoid = rrouter; } } - } - if (routerToBeAvoid == null) { - return this.start(router, user, caller, params, null); - } - // We would try best to deploy the router to another place - int retryIndex = 5; - ExcludeList[] avoids = new ExcludeList[5]; - avoids[0] = new ExcludeList(); - avoids[0].addPod(routerToBeAvoid.getPodIdToDeployIn()); - avoids[1] = new ExcludeList(); - avoids[1].addCluster(_hostDao.findById(routerToBeAvoid.getHostId()).getClusterId()); - avoids[2] = new ExcludeList(); - List volumes = _volumeDao.findByInstanceAndType(routerToBeAvoid.getId(), Type.ROOT); - if (volumes != null && volumes.size() != 0) { - avoids[2].addPool(volumes.get(0).getPoolId()); - } - avoids[2].addHost(routerToBeAvoid.getHostId()); - avoids[3] = new ExcludeList(); - avoids[3].addHost(routerToBeAvoid.getHostId()); - avoids[4] = new ExcludeList(); - - for (int i = 0; i < retryIndex; i++) { - if (s_logger.isTraceEnabled()) { - s_logger.trace("Try to deploy redundant virtual router:" + router.getHostName() + ", for " + i + " time"); - } - plan.setAvoids(avoids[i]); - try { - result = this.start(router, user, caller, params, plan); - } catch (InsufficientServerCapacityException ex) { - result = null; - } - if (result != null) { - break; - } - } - return result; - } - - @Override - public List deployVirtualRouterInGuestNetwork(Network guestNetwork, DeployDestination dest, Account owner, - Map params, boolean isRedundant) throws InsufficientCapacityException, - ConcurrentOperationException, ResourceUnavailableException { - - List routers = findOrDeployVirtualRouterInGuestNetwork - (guestNetwork, dest, owner, isRedundant, params); - - return startRouters(params, routers); - } - - protected List startRouters(Map params, List routers) throws StorageUnavailableException, InsufficientCapacityException, ConcurrentOperationException, - ResourceUnavailableException { - List runningRouters = null; - - if (routers != null) { - runningRouters = new ArrayList(); - } - - for (DomainRouterVO router : routers) { - boolean skip = false; - State state = router.getState(); - if (router.getHostId() != null && state != State.Running) { - HostVO host = _hostDao.findById(router.getHostId()); - if (host == null || host.getStatus() != Status.Up) { - skip = true; - } - } - if (!skip) { - if (state != State.Running) { - router = startVirtualRouter(router, _accountMgr.getSystemUser(), _accountMgr.getSystemAccount(), params); - } - if (router != null) { - runningRouters.add(router); - } - } - } - return runningRouters; - } - - @Override - public boolean finalizeVirtualMachineProfile(VirtualMachineProfile profile, DeployDestination dest, - ReservationContext context) { - - boolean dnsProvided = true; - boolean dhcpProvided = true; - boolean publicNetwork = false; - DataCenterVO dc = _dcDao.findById(dest.getDataCenter().getId()); - _dcDao.loadDetails(dc); - - //1) Set router details - DomainRouterVO router = profile.getVirtualMachine(); - Map details = _vmDetailsDao.findDetails(router.getId()); - router.setDetails(details); - - //2) Prepare boot loader elements related with Control network - - StringBuilder buf = profile.getBootArgsBuilder(); - buf.append(" template=domP"); - buf.append(" name=").append(profile.getHostName()); - - if (Boolean.valueOf(_configDao.getValue("system.vm.random.password"))) { - buf.append(" vmpassword=").append(_configDao.getValue("system.vm.password")); - } - - NicProfile controlNic = null; - String defaultDns1 = null; - String defaultDns2 = null; - for (NicProfile nic : profile.getNics()) { - int deviceId = nic.getDeviceId(); - buf.append(" eth").append(deviceId).append("ip=").append(nic.getIp4Address()); - buf.append(" eth").append(deviceId).append("mask=").append(nic.getNetmask()); - - if (nic.isDefaultNic()) { - buf.append(" gateway=").append(nic.getGateway()); - defaultDns1 = nic.getDns1(); - defaultDns2 = nic.getDns2(); - } - - if (nic.getTrafficType() == TrafficType.Management) { - buf.append(" localgw=").append(dest.getPod().getGateway()); - } else if (nic.getTrafficType() == TrafficType.Control) { - controlNic = nic; - // DOMR control command is sent over management server in VMware - if (dest.getHost().getHypervisorType() == HypervisorType.VMware) { - if (s_logger.isInfoEnabled()) { - s_logger.info("Check if we need to add management server explicit route to DomR. pod cidr: " - + dest.getPod().getCidrAddress() + "/" + dest.getPod().getCidrSize() - + ", pod gateway: " + dest.getPod().getGateway() + ", management host: " + _mgmt_host); - } - - if (s_logger.isInfoEnabled()) { - s_logger.info("Add management server explicit route to DomR."); - } - - // always add management explicit route, for basic networking setup, DomR may have two interfaces while both - // are on the same subnet - _mgmt_cidr = _configDao.getValue(Config.ManagementNetwork.key()); - if (NetUtils.isValidCIDR(_mgmt_cidr)) { - buf.append(" mgmtcidr=").append(_mgmt_cidr); - buf.append(" localgw=").append(dest.getPod().getGateway()); - } - - - if (dc.getNetworkType() == NetworkType.Basic) { - // ask domR to setup SSH on guest network - buf.append(" sshonguest=true"); - } - - } - } else if (nic.getTrafficType() == TrafficType.Guest) { - dnsProvided = _networkMgr.isProviderSupportServiceInNetwork(nic.getNetworkId(), Service.Dns, Provider.VirtualRouter); - dhcpProvided = _networkMgr.isProviderSupportServiceInNetwork(nic.getNetworkId(), Service.Dhcp, Provider.VirtualRouter); - //build bootloader parameter for the guest - buf.append(createGuestBootLoadArgs(nic, defaultDns1, defaultDns2, router)); - } else if (nic.getTrafficType() == TrafficType.Public) { - publicNetwork = true; - } - } - - if (controlNic == null) { - throw new CloudRuntimeException("Didn't start a control port"); - } - - String rpValue = _configDao.getValue(Config.NetworkRouterRpFilter.key()); - if (rpValue != null && rpValue.equalsIgnoreCase("true")) { - _disable_rp_filter = true; - }else { - _disable_rp_filter = false; - } - - String rpFilter = " "; - String type = null; - if (router.getVpcId() != null) { - type = "vpcrouter"; - if (_disable_rp_filter) { - rpFilter=" disable_rp_filter=true"; - } - } else if (!publicNetwork) { - type = "dhcpsrvr"; - } else { - type = "router"; - if (_disable_rp_filter) { - rpFilter=" disable_rp_filter=true"; - } - } - - if (_disable_rp_filter) { - rpFilter=" disable_rp_filter=true"; - } - - buf.append(" type=" + type + rpFilter); - - String domain_suffix = dc.getDetail(ZoneConfig.DnsSearchOrder.getName()); - if (domain_suffix != null) { - buf.append(" dnssearchorder=").append(domain_suffix); - } - - if (profile.getHypervisorType() == HypervisorType.VMware) { - buf.append(" extra_pubnics=" + _routerExtraPublicNics); - } - - /* If virtual router didn't provide DNS service but provide DHCP service, we need to override the DHCP response - * to return DNS server rather than - * virtual router itself. */ - if (dnsProvided || dhcpProvided) { - if (defaultDns1 != null) { - buf.append(" dns1=").append(defaultDns1); - } - if (defaultDns2 != null) { - buf.append(" dns2=").append(defaultDns2); - } - - boolean useExtDns = !dnsProvided; - /* For backward compatibility */ - String use_external_dns = _configDao.getValue(Config.UseExternalDnsServers.key()); - if (use_external_dns != null && use_external_dns.equals("true")) { - useExtDns = true; - } - - if (useExtDns) { - buf.append(" useextdns=true"); - } - } - - if (s_logger.isDebugEnabled()) { - s_logger.debug("Boot Args for " + profile + ": " + buf.toString()); - } - - return true; - } - - - protected StringBuilder createGuestBootLoadArgs(NicProfile guestNic, String defaultDns1, - String defaultDns2, DomainRouterVO router) { - long guestNetworkId = guestNic.getNetworkId(); - NetworkVO guestNetwork = _networkDao.findById(guestNetworkId); - String dhcpRange = null; - DataCenterVO dc = _dcDao.findById(guestNetwork.getDataCenterId()); - - StringBuilder buf = new StringBuilder(); - - boolean isRedundant = router.getIsRedundantRouter(); - if (isRedundant) { - buf.append(" redundant_router=1"); - List routers = _routerDao.listByNetworkAndRole(guestNetwork.getId(), Role.VIRTUAL_ROUTER); - try { - int priority = getUpdatedPriority(guestNetwork, routers, router); - router.setPriority(priority); - } catch (InsufficientVirtualNetworkCapcityException e) { - s_logger.error("Failed to get update priority!", e); - throw new CloudRuntimeException("Failed to get update priority!"); - } - Network net = _networkMgr.getNetwork(guestNic.getNetworkId()); - buf.append(" guestgw=").append(net.getGateway()); - String brd = NetUtils.long2Ip(NetUtils.ip2Long(guestNic.getIp4Address()) | ~NetUtils.ip2Long(guestNic.getNetmask())); - buf.append(" guestbrd=").append(brd); - buf.append(" guestcidrsize=").append(NetUtils.getCidrSize(guestNic.getNetmask())); - buf.append(" router_pr=").append(router.getPriority()); - } - - //setup network domain - String domain = guestNetwork.getNetworkDomain(); - if (domain != null) { - buf.append(" domain=" + domain); - } - - //setup dhcp range - if (dc.getNetworkType() == NetworkType.Basic) { - if (guestNic.isDefaultNic()) { - long cidrSize = NetUtils.getCidrSize(guestNic.getNetmask()); - String cidr = NetUtils.getCidrSubNet(guestNic.getGateway(), cidrSize); - if (cidr != null) { - dhcpRange = NetUtils.getIpRangeStartIpFromCidr(cidr, cidrSize); - } - } - } else if (dc.getNetworkType() == NetworkType.Advanced) { - String cidr = guestNetwork.getCidr(); - if (cidr != null) { - dhcpRange = NetUtils.getDhcpRange(cidr); - } - } - - if (dhcpRange != null) { - buf.append(" dhcprange=" + dhcpRange); - } - - return buf; - } - - - protected String getGuestDhcpRange(NicProfile guestNic, Network guestNetwork, DataCenter dc) { - String dhcpRange = null; - //setup dhcp range - if (dc.getNetworkType() == NetworkType.Basic) { - long cidrSize = NetUtils.getCidrSize(guestNic.getNetmask()); - String cidr = NetUtils.getCidrSubNet(guestNic.getGateway(), cidrSize); - if (cidr != null) { - dhcpRange = NetUtils.getIpRangeStartIpFromCidr(cidr, cidrSize); - } - } else if (dc.getNetworkType() == NetworkType.Advanced) { - String cidr = guestNetwork.getCidr(); - if (cidr != null) { - dhcpRange = NetUtils.getDhcpRange(cidr); - } - } - return dhcpRange; - } - - @Override - public boolean finalizeDeployment(Commands cmds, VirtualMachineProfile profile, - DeployDestination dest, ReservationContext context) throws ResourceUnavailableException { - DomainRouterVO router = profile.getVirtualMachine(); - - List nics = profile.getNics(); - for (NicProfile nic : nics) { - if (nic.getTrafficType() == TrafficType.Public) { - router.setPublicIpAddress(nic.getIp4Address()); - router.setPublicNetmask(nic.getNetmask()); - router.setPublicMacAddress(nic.getMacAddress()); - } else if (nic.getTrafficType() == TrafficType.Control) { - router.setPrivateIpAddress(nic.getIp4Address()); - router.setPrivateMacAddress(nic.getMacAddress()); - } - } - _routerDao.update(router.getId(), router); - - finalizeCommandsOnStart(cmds, profile); - return true; - } - - @Override - public boolean finalizeCommandsOnStart(Commands cmds, VirtualMachineProfile profile) { - DomainRouterVO router = profile.getVirtualMachine(); - NicProfile controlNic = getControlNic(profile); - - if (controlNic == null) { - s_logger.error("Control network doesn't exist for the router " + router); - return false; - } - - finalizeSshAndVersionAndNetworkUsageOnStart(cmds, profile, router, controlNic); - - // restart network if restartNetwork = false is not specified in profile parameters - boolean reprogramGuestNtwks = true; - if (profile.getParameter(Param.ReProgramGuestNetworks) != null - && (Boolean) profile.getParameter(Param.ReProgramGuestNetworks) == false) { - reprogramGuestNtwks = false; - } - - VirtualRouterProvider vrProvider = _vrProviderDao.findById(router.getElementId()); - if (vrProvider == null) { - throw new CloudRuntimeException("Cannot find related virtual router provider of router: " + router.getHostName()); - } - Provider provider = Network.Provider.getProvider(vrProvider.getType().toString()); - if (provider == null) { - throw new CloudRuntimeException("Cannot find related provider of virtual router provider: " + vrProvider.getType().toString()); - } - - List routerGuestNtwkIds = _routerDao.getRouterNetworks(router.getId()); - for (Long guestNetworkId : routerGuestNtwkIds) { - if (reprogramGuestNtwks) { - finalizeIpAssocForNetwork(cmds, router, provider, guestNetworkId, null); - finalizeNetworkRulesForNetwork(cmds, router, provider, guestNetworkId); - } - - finalizeUserDataAndDhcpOnStart(cmds, router, provider, guestNetworkId); - } - - return true; - } - - protected NicProfile getControlNic(VirtualMachineProfile profile) { - DomainRouterVO router = profile.getVirtualMachine(); - DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); - NicProfile controlNic = null; - if (profile.getHypervisorType() == HypervisorType.VMware && dcVo.getNetworkType() == NetworkType.Basic) { - // TODO this is a ugly to test hypervisor type here - // for basic network mode, we will use the guest NIC for control NIC - for (NicProfile nic : profile.getNics()) { - if (nic.getTrafficType() == TrafficType.Guest && nic.getIp4Address() != null) { - controlNic = nic; - } - } - } else { - for (NicProfile nic : profile.getNics()) { - if (nic.getTrafficType() == TrafficType.Control && nic.getIp4Address() != null) { - controlNic = nic; - } - } - } - return controlNic; - } - - protected void finalizeSshAndVersionAndNetworkUsageOnStart(Commands cmds, VirtualMachineProfile profile, DomainRouterVO router, NicProfile controlNic) { - cmds.addCommand("checkSsh", new CheckSshCommand(profile.getInstanceName(), controlNic.getIp4Address(), 3922)); - - // Update router template/scripts version - final GetDomRVersionCmd command = new GetDomRVersionCmd(); - command.setAccessDetail(NetworkElementCommand.ROUTER_IP, controlNic.getIp4Address()); - command.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); - cmds.addCommand("getDomRVersion", command); - - // Network usage command to create iptables rules - boolean forVpc = profile.getVirtualMachine().getVpcId() != null; - cmds.addCommand("networkUsage", new NetworkUsageCommand(controlNic.getIp4Address(), router.getHostName(), "create", forVpc)); - } - - protected void finalizeUserDataAndDhcpOnStart(Commands cmds, DomainRouterVO router, Provider provider, Long guestNetworkId) { - if (_networkMgr.isProviderSupportServiceInNetwork(guestNetworkId, Service.Dhcp, provider)) { - // Resend dhcp - s_logger.debug("Reapplying dhcp entries as a part of domR " + router + " start..."); - createDhcpEntryCommandsForVMs(router, cmds, guestNetworkId); - } - - if (_networkMgr.isProviderSupportServiceInNetwork(guestNetworkId, Service.UserData, provider)) { - // Resend user data - s_logger.debug("Reapplying vm data (userData and metaData) entries as a part of domR " + router + " start..."); - createVmDataCommandForVMs(router, cmds, guestNetworkId); - } - } - - protected void finalizeNetworkRulesForNetwork(Commands cmds, DomainRouterVO router, Provider provider, Long guestNetworkId) { - s_logger.debug("Resending ipAssoc, port forwarding, load balancing rules as a part of Virtual router start"); - - ArrayList publicIps = getPublicIpsToApply(router, provider, guestNetworkId); - - if (publicIps != null && !publicIps.isEmpty()) { - List vpns = new ArrayList(); - List pfRules = new ArrayList(); - List staticNatFirewallRules = new ArrayList(); - List staticNats = new ArrayList(); - List firewallRules = new ArrayList(); - - //Get information about all the rules (StaticNats and StaticNatRules; PFVPN to reapply on domR start) - for (PublicIpAddress ip : publicIps) { - if (_networkMgr.isProviderSupportServiceInNetwork(guestNetworkId, Service.PortForwarding, provider)) { - pfRules.addAll(_pfRulesDao.listForApplication(ip.getId())); - } - if (_networkMgr.isProviderSupportServiceInNetwork(guestNetworkId, Service.StaticNat, provider)) { - staticNatFirewallRules.addAll(_rulesDao.listByIpAndPurpose(ip.getId(), Purpose.StaticNat)); - } - if (_networkMgr.isProviderSupportServiceInNetwork(guestNetworkId, Service.Firewall, provider)) { - firewallRules.addAll(_rulesDao.listByIpAndPurpose(ip.getId(), Purpose.Firewall)); - } - - if (_networkMgr.isProviderSupportServiceInNetwork(guestNetworkId, Service.Vpn, provider)) { - RemoteAccessVpn vpn = _vpnDao.findById(ip.getId()); - if (vpn != null) { - vpns.add(vpn); - } - } - - if (_networkMgr.isProviderSupportServiceInNetwork(guestNetworkId, Service.StaticNat, provider)) { - if (ip.isOneToOneNat()) { - String dstIp = _networkMgr.getIpInNetwork(ip.getAssociatedWithVmId(), guestNetworkId); - StaticNatImpl staticNat = new StaticNatImpl(ip.getAccountId(), ip.getDomainId(), guestNetworkId, ip.getId(), dstIp, false); - staticNats.add(staticNat); - } - } - } - - //Re-apply static nats - s_logger.debug("Found " + staticNats.size() + " static nat(s) to apply as a part of domR " + router + " start."); - if (!staticNats.isEmpty()) { - createApplyStaticNatCommands(staticNats, router, cmds, guestNetworkId); - } - - //Re-apply firewall rules - s_logger.debug("Found " + staticNats.size() + " firewall rule(s) to apply as a part of domR " + router + " start."); - if (!firewallRules.isEmpty()) { - createFirewallRulesCommands(firewallRules, router, cmds, guestNetworkId); - } - - // Re-apply port forwarding rules - s_logger.debug("Found " + pfRules.size() + " port forwarding rule(s) to apply as a part of domR " + router + " start."); - if (!pfRules.isEmpty()) { - createApplyPortForwardingRulesCommands(pfRules, router, cmds, guestNetworkId); - } - - // Re-apply static nat rules - s_logger.debug("Found " + staticNatFirewallRules.size() + " static nat rule(s) to apply as a part of domR " + router + " start."); - if (!staticNatFirewallRules.isEmpty()) { - List staticNatRules = new ArrayList(); - for (FirewallRule rule : staticNatFirewallRules) { - staticNatRules.add(_rulesMgr.buildStaticNatRule(rule, false)); - } - createApplyStaticNatRulesCommands(staticNatRules, router, cmds, guestNetworkId); - } - - // Re-apply vpn rules - s_logger.debug("Found " + vpns.size() + " vpn(s) to apply as a part of domR " + router + " start."); - if (!vpns.isEmpty()) { - for (RemoteAccessVpn vpn : vpns) { - createApplyVpnCommands(vpn, router, cmds); - } - } - - List lbs = _loadBalancerDao.listByNetworkId(guestNetworkId); - List lbRules = new ArrayList(); - if (_networkMgr.isProviderSupportServiceInNetwork(guestNetworkId, Service.Lb, provider)) { - // Re-apply load balancing rules - for (LoadBalancerVO lb : lbs) { - List dstList = _lbMgr.getExistingDestinations(lb.getId()); - List policyList = _lbMgr.getStickinessPolicies(lb.getId()); - LoadBalancingRule loadBalancing = new LoadBalancingRule(lb, dstList, policyList); - lbRules.add(loadBalancing); - } - } - - s_logger.debug("Found " + lbRules.size() + " load balancing rule(s) to apply as a part of domR " + router + " start."); - if (!lbRules.isEmpty()) { - createApplyLoadBalancingRulesCommands(lbRules, router, cmds, guestNetworkId); - } - } - } - - protected void finalizeIpAssocForNetwork(Commands cmds, VirtualRouter router, Provider provider, - Long guestNetworkId, Map vlanMacAddress) { - - ArrayList publicIps = getPublicIpsToApply(router, provider, guestNetworkId); - - if (publicIps != null && !publicIps.isEmpty()) { - s_logger.debug("Found " + publicIps.size() + " ip(s) to apply as a part of domR " + router + " start."); - // Re-apply public ip addresses - should come before PF/LB/VPN - if (_networkMgr.isProviderSupportServiceInNetwork(guestNetworkId, Service.Firewall, provider)) { - createAssociateIPCommands(router, publicIps, cmds, 0); - } - } - } - - protected ArrayList getPublicIpsToApply(VirtualRouter router, Provider provider, - Long guestNetworkId, com.cloud.network.IpAddress.State... skipInStates) { - long ownerId = router.getAccountId(); - final List userIps = _networkMgr.listPublicIpsAssignedToGuestNtwk(ownerId, guestNetworkId, null); - List allPublicIps = new ArrayList(); - if (userIps != null && !userIps.isEmpty()) { - boolean addIp = true; - for (IPAddressVO userIp : userIps) { - if (skipInStates != null) { - for (IpAddress.State stateToSkip : skipInStates) { - if (userIp.getState() == stateToSkip) { - s_logger.debug("Skipping ip address " + userIp + " in state " + userIp.getState()); - addIp = false; - break; - } - } - } - - if (addIp) { - PublicIp publicIp = new PublicIp(userIp, _vlanDao.findById(userIp.getVlanId()), - NetUtils.createSequenceBasedMacAddress(userIp.getMacAddress())); - allPublicIps.add(publicIp); - } - } - } - - //Get public Ips that should be handled by router - Network network = _networkDao.findById(guestNetworkId); - Map> ipToServices = _networkMgr.getIpToServices(allPublicIps, false, true); - Map> providerToIpList = _networkMgr.getProviderToIpList(network, ipToServices); - // Only cover virtual router for now, if ELB use it this need to be modified - - ArrayList publicIps = providerToIpList.get(provider); - return publicIps; - } - - @Override - public boolean finalizeStart(VirtualMachineProfile profile, long hostId, Commands cmds, - ReservationContext context) { - DomainRouterVO router = profile.getVirtualMachine(); - - boolean result = true; - - Answer answer = cmds.getAnswer("checkSsh"); - if (answer != null && answer instanceof CheckSshAnswer) { - CheckSshAnswer sshAnswer = (CheckSshAnswer) answer; - if (sshAnswer == null || !sshAnswer.getResult()) { - s_logger.warn("Unable to ssh to the VM: " + sshAnswer.getDetails()); - result = false; - } - } else { - result = false; - } - if (result == false) { - return result; - } - - //Get guest networks info - List guestNetworks = new ArrayList(); - - List routerNics = _nicDao.listByVmId(profile.getId()); - for (Nic routerNic : routerNics) { - Network network = _networkMgr.getNetwork(routerNic.getNetworkId()); - if (network.getTrafficType() == TrafficType.Guest) { - guestNetworks.add(network); - } - } - - answer = cmds.getAnswer("getDomRVersion"); - if (answer != null && answer instanceof GetDomRVersionAnswer) { - GetDomRVersionAnswer versionAnswer = (GetDomRVersionAnswer)answer; - if (answer == null || !answer.getResult()) { - s_logger.warn("Unable to get the template/scripts version of router " + router.getInstanceName() + - " due to: " + versionAnswer.getDetails()); - result = false; - } else { - router.setTemplateVersion(versionAnswer.getTemplateVersion()); - router.setScriptsVersion(versionAnswer.getScriptsVersion()); - router = _routerDao.persist(router, guestNetworks); - } - } else { - result = false; - } - - return result; - } - - @Override - public void finalizeStop(VirtualMachineProfile profile, StopAnswer answer) { - if (answer != null) { - VMInstanceVO vm = profile.getVirtualMachine(); - DomainRouterVO domR = _routerDao.findById(vm.getId()); - processStopOrRebootAnswer(domR, answer); - } - } - - @Override - public void finalizeExpunge(DomainRouterVO vm) { - } - - - @Override - public boolean startRemoteAccessVpn(Network network, RemoteAccessVpn vpn, List routers) - throws ResourceUnavailableException { - if (routers == null || routers.isEmpty()) { - s_logger.warn("Failed to start remote access VPN: no router found for account and zone"); - throw new ResourceUnavailableException("Failed to start remote access VPN: no router found for account and zone", - DataCenter.class, network.getDataCenterId()); - } - - for (VirtualRouter router : routers) { - if (router.getState() != State.Running) { - s_logger.warn("Failed to start remote access VPN: router not in right state " + router.getState()); - throw new ResourceUnavailableException("Failed to start remote access VPN: router not in right state " - + router.getState(), DataCenter.class, network.getDataCenterId()); - } - - Commands cmds = new Commands(OnError.Stop); - createApplyVpnCommands(vpn, router, cmds); - - try { - _agentMgr.send(router.getHostId(), cmds); - } catch (OperationTimedoutException e) { - s_logger.debug("Failed to start remote access VPN: ", e); - throw new AgentUnavailableException("Unable to send commands to virtual router ", router.getHostId(), e); - } - Answer answer = cmds.getAnswer("users"); - if (!answer.getResult()) { - s_logger.error("Unable to start vpn: unable add users to vpn in zone " + router.getDataCenterIdToDeployIn() - + " for account " + vpn.getAccountId() + " on domR: " + router.getInstanceName() - + " due to " + answer.getDetails()); - throw new ResourceUnavailableException("Unable to start vpn: Unable to add users to vpn in zone " + - router.getDataCenterIdToDeployIn() + " for account " + vpn.getAccountId() + " on domR: " - + router.getInstanceName() + " due to " + answer.getDetails(), DataCenter.class, router.getDataCenterIdToDeployIn()); - } - answer = cmds.getAnswer("startVpn"); - if (!answer.getResult()) { - s_logger.error("Unable to start vpn in zone " + router.getDataCenterIdToDeployIn() + " for account " + - vpn.getAccountId() + " on domR: " + router.getInstanceName() + " due to " - + answer.getDetails()); - throw new ResourceUnavailableException("Unable to start vpn in zone " + router.getDataCenterIdToDeployIn() - + " for account " + vpn.getAccountId() + " on domR: " + router.getInstanceName() - + " due to " + answer.getDetails(), DataCenter.class, router.getDataCenterIdToDeployIn()); - } - - } - return true; - } - - - @Override - public boolean deleteRemoteAccessVpn(Network network, RemoteAccessVpn vpn, List routers) - throws ResourceUnavailableException { - if (routers == null || routers.isEmpty()) { - s_logger.warn("Failed to delete remote access VPN: no router found for account and zone"); - throw new ResourceUnavailableException("Failed to delete remote access VPN", DataCenter.class, network.getDataCenterId()); - } - - boolean result = true; - for (VirtualRouter router : routers) { - if (router.getState() == State.Running) { - Commands cmds = new Commands(OnError.Continue); - IpAddress ip = _networkMgr.getIp(vpn.getServerAddressId()); - - RemoteAccessVpnCfgCommand removeVpnCmd = new RemoteAccessVpnCfgCommand(false, ip.getAddress().addr(), - vpn.getLocalIp(), vpn.getIpRange(), vpn.getIpsecPresharedKey()); - removeVpnCmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); - removeVpnCmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(network.getId(), router.getId())); - removeVpnCmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); - - DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); - removeVpnCmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); - - cmds.addCommand(removeVpnCmd); - - result = result && sendCommandsToRouter(router, cmds); - } else if (router.getState() == State.Stopped) { - s_logger.debug("Router " + router + " is in Stopped state, not sending deleteRemoteAccessVpn command to it"); - continue; - } else { - s_logger.warn("Failed to delete remote access VPN: domR " + router + " is not in right state " + router.getState()); - throw new ResourceUnavailableException("Failed to delete remote access VPN: domR is not in right state " + - router.getState(), DataCenter.class, network.getDataCenterId()); - } - } - - return result; - } - - - private DomainRouterVO start(DomainRouterVO router, User user, Account caller, Map params, DeploymentPlan planToDeploy) - throws StorageUnavailableException, InsufficientCapacityException, - ConcurrentOperationException, ResourceUnavailableException { - s_logger.debug("Starting router " + router); - if (_itMgr.start(router, params, user, caller, planToDeploy) != null) { - // We don't want the failure of VPN Connection affect the status of router, so we try to make connection only after router start successfully + } + if (routerToBeAvoid == null) { + return this.start(router, user, caller, params, null); + } + // We would try best to deploy the router to another place + int retryIndex = 5; + ExcludeList[] avoids = new ExcludeList[5]; + avoids[0] = new ExcludeList(); + avoids[0].addPod(routerToBeAvoid.getPodIdToDeployIn()); + avoids[1] = new ExcludeList(); + avoids[1].addCluster(_hostDao.findById(routerToBeAvoid.getHostId()).getClusterId()); + avoids[2] = new ExcludeList(); + List volumes = _volumeDao.findByInstanceAndType(routerToBeAvoid.getId(), Type.ROOT); + if (volumes != null && volumes.size() != 0) { + avoids[2].addPool(volumes.get(0).getPoolId()); + } + avoids[2].addHost(routerToBeAvoid.getHostId()); + avoids[3] = new ExcludeList(); + avoids[3].addHost(routerToBeAvoid.getHostId()); + avoids[4] = new ExcludeList(); + + for (int i = 0; i < retryIndex; i++) { + if (s_logger.isTraceEnabled()) { + s_logger.trace("Try to deploy redundant virtual router:" + router.getHostName() + ", for " + i + " time"); + } + plan.setAvoids(avoids[i]); + try { + result = this.start(router, user, caller, params, plan); + } catch (InsufficientServerCapacityException ex) { + result = null; + } + if (result != null) { + break; + } + } + return result; + } + + @Override + public List deployVirtualRouterInGuestNetwork(Network guestNetwork, DeployDestination dest, Account owner, + Map params, boolean isRedundant) throws InsufficientCapacityException, + ConcurrentOperationException, ResourceUnavailableException { + + List routers = findOrDeployVirtualRouterInGuestNetwork + (guestNetwork, dest, owner, isRedundant, params); + + return startRouters(params, routers); + } + + protected List startRouters(Map params, List routers) throws StorageUnavailableException, InsufficientCapacityException, ConcurrentOperationException, + ResourceUnavailableException { + List runningRouters = null; + + if (routers != null) { + runningRouters = new ArrayList(); + } + + for (DomainRouterVO router : routers) { + boolean skip = false; + State state = router.getState(); + if (router.getHostId() != null && state != State.Running) { + HostVO host = _hostDao.findById(router.getHostId()); + if (host == null || host.getStatus() != Status.Up) { + skip = true; + } + } + if (!skip) { + if (state != State.Running) { + router = startVirtualRouter(router, _accountMgr.getSystemUser(), _accountMgr.getSystemAccount(), params); + } + if (router != null) { + runningRouters.add(router); + } + } + } + return runningRouters; + } + + @Override + public boolean finalizeVirtualMachineProfile(VirtualMachineProfile profile, DeployDestination dest, + ReservationContext context) { + + boolean dnsProvided = true; + boolean dhcpProvided = true; + boolean publicNetwork = false; + DataCenterVO dc = _dcDao.findById(dest.getDataCenter().getId()); + _dcDao.loadDetails(dc); + + //1) Set router details + DomainRouterVO router = profile.getVirtualMachine(); + Map details = _vmDetailsDao.findDetails(router.getId()); + router.setDetails(details); + + //2) Prepare boot loader elements related with Control network + + StringBuilder buf = profile.getBootArgsBuilder(); + buf.append(" template=domP"); + buf.append(" name=").append(profile.getHostName()); + + if (Boolean.valueOf(_configDao.getValue("system.vm.random.password"))) { + buf.append(" vmpassword=").append(_configDao.getValue("system.vm.password")); + } + + NicProfile controlNic = null; + String defaultDns1 = null; + String defaultDns2 = null; + for (NicProfile nic : profile.getNics()) { + int deviceId = nic.getDeviceId(); + buf.append(" eth").append(deviceId).append("ip=").append(nic.getIp4Address()); + buf.append(" eth").append(deviceId).append("mask=").append(nic.getNetmask()); + + if (nic.isDefaultNic()) { + buf.append(" gateway=").append(nic.getGateway()); + defaultDns1 = nic.getDns1(); + defaultDns2 = nic.getDns2(); + } + + if (nic.getTrafficType() == TrafficType.Management) { + buf.append(" localgw=").append(dest.getPod().getGateway()); + } else if (nic.getTrafficType() == TrafficType.Control) { + controlNic = nic; + // DOMR control command is sent over management server in VMware + if (dest.getHost().getHypervisorType() == HypervisorType.VMware) { + if (s_logger.isInfoEnabled()) { + s_logger.info("Check if we need to add management server explicit route to DomR. pod cidr: " + + dest.getPod().getCidrAddress() + "/" + dest.getPod().getCidrSize() + + ", pod gateway: " + dest.getPod().getGateway() + ", management host: " + _mgmt_host); + } + + if (s_logger.isInfoEnabled()) { + s_logger.info("Add management server explicit route to DomR."); + } + + // always add management explicit route, for basic networking setup, DomR may have two interfaces while both + // are on the same subnet + _mgmt_cidr = _configDao.getValue(Config.ManagementNetwork.key()); + if (NetUtils.isValidCIDR(_mgmt_cidr)) { + buf.append(" mgmtcidr=").append(_mgmt_cidr); + buf.append(" localgw=").append(dest.getPod().getGateway()); + } + + + if (dc.getNetworkType() == NetworkType.Basic) { + // ask domR to setup SSH on guest network + buf.append(" sshonguest=true"); + } + + } + } else if (nic.getTrafficType() == TrafficType.Guest) { + dnsProvided = _networkMgr.isProviderSupportServiceInNetwork(nic.getNetworkId(), Service.Dns, Provider.VirtualRouter); + dhcpProvided = _networkMgr.isProviderSupportServiceInNetwork(nic.getNetworkId(), Service.Dhcp, Provider.VirtualRouter); + //build bootloader parameter for the guest + buf.append(createGuestBootLoadArgs(nic, defaultDns1, defaultDns2, router)); + } else if (nic.getTrafficType() == TrafficType.Public) { + publicNetwork = true; + } + } + + if (controlNic == null) { + throw new CloudRuntimeException("Didn't start a control port"); + } + + String rpValue = _configDao.getValue(Config.NetworkRouterRpFilter.key()); + if (rpValue != null && rpValue.equalsIgnoreCase("true")) { + _disable_rp_filter = true; + }else { + _disable_rp_filter = false; + } + + String rpFilter = " "; + String type = null; + if (router.getVpcId() != null) { + type = "vpcrouter"; + if (_disable_rp_filter) { + rpFilter=" disable_rp_filter=true"; + } + } else if (!publicNetwork) { + type = "dhcpsrvr"; + } else { + type = "router"; + if (_disable_rp_filter) { + rpFilter=" disable_rp_filter=true"; + } + } + + if (_disable_rp_filter) { + rpFilter=" disable_rp_filter=true"; + } + + buf.append(" type=" + type + rpFilter); + + String domain_suffix = dc.getDetail(ZoneConfig.DnsSearchOrder.getName()); + if (domain_suffix != null) { + buf.append(" dnssearchorder=").append(domain_suffix); + } + + if (profile.getHypervisorType() == HypervisorType.VMware) { + buf.append(" extra_pubnics=" + _routerExtraPublicNics); + } + + /* If virtual router didn't provide DNS service but provide DHCP service, we need to override the DHCP response + * to return DNS server rather than + * virtual router itself. */ + if (dnsProvided || dhcpProvided) { + if (defaultDns1 != null) { + buf.append(" dns1=").append(defaultDns1); + } + if (defaultDns2 != null) { + buf.append(" dns2=").append(defaultDns2); + } + + boolean useExtDns = !dnsProvided; + /* For backward compatibility */ + String use_external_dns = _configDao.getValue(Config.UseExternalDnsServers.key()); + if (use_external_dns != null && use_external_dns.equals("true")) { + useExtDns = true; + } + + if (useExtDns) { + buf.append(" useextdns=true"); + } + } + + if (s_logger.isDebugEnabled()) { + s_logger.debug("Boot Args for " + profile + ": " + buf.toString()); + } + + return true; + } + + + protected StringBuilder createGuestBootLoadArgs(NicProfile guestNic, String defaultDns1, + String defaultDns2, DomainRouterVO router) { + long guestNetworkId = guestNic.getNetworkId(); + NetworkVO guestNetwork = _networkDao.findById(guestNetworkId); + String dhcpRange = null; + DataCenterVO dc = _dcDao.findById(guestNetwork.getDataCenterId()); + + StringBuilder buf = new StringBuilder(); + + boolean isRedundant = router.getIsRedundantRouter(); + if (isRedundant) { + buf.append(" redundant_router=1"); + List routers = _routerDao.listByNetworkAndRole(guestNetwork.getId(), Role.VIRTUAL_ROUTER); + try { + int priority = getUpdatedPriority(guestNetwork, routers, router); + router.setPriority(priority); + } catch (InsufficientVirtualNetworkCapcityException e) { + s_logger.error("Failed to get update priority!", e); + throw new CloudRuntimeException("Failed to get update priority!"); + } + Network net = _networkMgr.getNetwork(guestNic.getNetworkId()); + buf.append(" guestgw=").append(net.getGateway()); + String brd = NetUtils.long2Ip(NetUtils.ip2Long(guestNic.getIp4Address()) | ~NetUtils.ip2Long(guestNic.getNetmask())); + buf.append(" guestbrd=").append(brd); + buf.append(" guestcidrsize=").append(NetUtils.getCidrSize(guestNic.getNetmask())); + buf.append(" router_pr=").append(router.getPriority()); + } + + //setup network domain + String domain = guestNetwork.getNetworkDomain(); + if (domain != null) { + buf.append(" domain=" + domain); + } + + //setup dhcp range + if (dc.getNetworkType() == NetworkType.Basic) { + if (guestNic.isDefaultNic()) { + long cidrSize = NetUtils.getCidrSize(guestNic.getNetmask()); + String cidr = NetUtils.getCidrSubNet(guestNic.getGateway(), cidrSize); + if (cidr != null) { + dhcpRange = NetUtils.getIpRangeStartIpFromCidr(cidr, cidrSize); + } + } + } else if (dc.getNetworkType() == NetworkType.Advanced) { + String cidr = guestNetwork.getCidr(); + if (cidr != null) { + dhcpRange = NetUtils.getDhcpRange(cidr); + } + } + + if (dhcpRange != null) { + buf.append(" dhcprange=" + dhcpRange); + } + + return buf; + } + + + protected String getGuestDhcpRange(NicProfile guestNic, Network guestNetwork, DataCenter dc) { + String dhcpRange = null; + //setup dhcp range + if (dc.getNetworkType() == NetworkType.Basic) { + long cidrSize = NetUtils.getCidrSize(guestNic.getNetmask()); + String cidr = NetUtils.getCidrSubNet(guestNic.getGateway(), cidrSize); + if (cidr != null) { + dhcpRange = NetUtils.getIpRangeStartIpFromCidr(cidr, cidrSize); + } + } else if (dc.getNetworkType() == NetworkType.Advanced) { + String cidr = guestNetwork.getCidr(); + if (cidr != null) { + dhcpRange = NetUtils.getDhcpRange(cidr); + } + } + return dhcpRange; + } + + @Override + public boolean finalizeDeployment(Commands cmds, VirtualMachineProfile profile, + DeployDestination dest, ReservationContext context) throws ResourceUnavailableException { + DomainRouterVO router = profile.getVirtualMachine(); + + List nics = profile.getNics(); + for (NicProfile nic : nics) { + if (nic.getTrafficType() == TrafficType.Public) { + router.setPublicIpAddress(nic.getIp4Address()); + router.setPublicNetmask(nic.getNetmask()); + router.setPublicMacAddress(nic.getMacAddress()); + } else if (nic.getTrafficType() == TrafficType.Control) { + router.setPrivateIpAddress(nic.getIp4Address()); + router.setPrivateMacAddress(nic.getMacAddress()); + } + } + _routerDao.update(router.getId(), router); + + finalizeCommandsOnStart(cmds, profile); + return true; + } + + @Override + public boolean finalizeCommandsOnStart(Commands cmds, VirtualMachineProfile profile) { + DomainRouterVO router = profile.getVirtualMachine(); + NicProfile controlNic = getControlNic(profile); + + if (controlNic == null) { + s_logger.error("Control network doesn't exist for the router " + router); + return false; + } + + finalizeSshAndVersionAndNetworkUsageOnStart(cmds, profile, router, controlNic); + + // restart network if restartNetwork = false is not specified in profile parameters + boolean reprogramGuestNtwks = true; + if (profile.getParameter(Param.ReProgramGuestNetworks) != null + && (Boolean) profile.getParameter(Param.ReProgramGuestNetworks) == false) { + reprogramGuestNtwks = false; + } + + VirtualRouterProvider vrProvider = _vrProviderDao.findById(router.getElementId()); + if (vrProvider == null) { + throw new CloudRuntimeException("Cannot find related virtual router provider of router: " + router.getHostName()); + } + Provider provider = Network.Provider.getProvider(vrProvider.getType().toString()); + if (provider == null) { + throw new CloudRuntimeException("Cannot find related provider of virtual router provider: " + vrProvider.getType().toString()); + } + + List routerGuestNtwkIds = _routerDao.getRouterNetworks(router.getId()); + for (Long guestNetworkId : routerGuestNtwkIds) { + if (reprogramGuestNtwks) { + finalizeIpAssocForNetwork(cmds, router, provider, guestNetworkId, null); + finalizeNetworkRulesForNetwork(cmds, router, provider, guestNetworkId); + } + + finalizeUserDataAndDhcpOnStart(cmds, router, provider, guestNetworkId); + } + + return true; + } + + protected NicProfile getControlNic(VirtualMachineProfile profile) { + DomainRouterVO router = profile.getVirtualMachine(); + DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); + NicProfile controlNic = null; + if (profile.getHypervisorType() == HypervisorType.VMware && dcVo.getNetworkType() == NetworkType.Basic) { + // TODO this is a ugly to test hypervisor type here + // for basic network mode, we will use the guest NIC for control NIC + for (NicProfile nic : profile.getNics()) { + if (nic.getTrafficType() == TrafficType.Guest && nic.getIp4Address() != null) { + controlNic = nic; + } + } + } else { + for (NicProfile nic : profile.getNics()) { + if (nic.getTrafficType() == TrafficType.Control && nic.getIp4Address() != null) { + controlNic = nic; + } + } + } + return controlNic; + } + + protected void finalizeSshAndVersionAndNetworkUsageOnStart(Commands cmds, VirtualMachineProfile profile, DomainRouterVO router, NicProfile controlNic) { + cmds.addCommand("checkSsh", new CheckSshCommand(profile.getInstanceName(), controlNic.getIp4Address(), 3922)); + + // Update router template/scripts version + final GetDomRVersionCmd command = new GetDomRVersionCmd(); + command.setAccessDetail(NetworkElementCommand.ROUTER_IP, controlNic.getIp4Address()); + command.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); + cmds.addCommand("getDomRVersion", command); + + // Network usage command to create iptables rules + boolean forVpc = profile.getVirtualMachine().getVpcId() != null; + cmds.addCommand("networkUsage", new NetworkUsageCommand(controlNic.getIp4Address(), router.getHostName(), "create", forVpc)); + } + + protected void finalizeUserDataAndDhcpOnStart(Commands cmds, DomainRouterVO router, Provider provider, Long guestNetworkId) { + if (_networkMgr.isProviderSupportServiceInNetwork(guestNetworkId, Service.Dhcp, provider)) { + // Resend dhcp + s_logger.debug("Reapplying dhcp entries as a part of domR " + router + " start..."); + createDhcpEntryCommandsForVMs(router, cmds, guestNetworkId); + } + + if (_networkMgr.isProviderSupportServiceInNetwork(guestNetworkId, Service.UserData, provider)) { + // Resend user data + s_logger.debug("Reapplying vm data (userData and metaData) entries as a part of domR " + router + " start..."); + createVmDataCommandForVMs(router, cmds, guestNetworkId); + } + } + + protected void finalizeNetworkRulesForNetwork(Commands cmds, DomainRouterVO router, Provider provider, Long guestNetworkId) { + s_logger.debug("Resending ipAssoc, port forwarding, load balancing rules as a part of Virtual router start"); + + ArrayList publicIps = getPublicIpsToApply(router, provider, guestNetworkId); + + if (publicIps != null && !publicIps.isEmpty()) { + List vpns = new ArrayList(); + List pfRules = new ArrayList(); + List staticNatFirewallRules = new ArrayList(); + List staticNats = new ArrayList(); + List firewallRules = new ArrayList(); + + //Get information about all the rules (StaticNats and StaticNatRules; PFVPN to reapply on domR start) + for (PublicIpAddress ip : publicIps) { + if (_networkMgr.isProviderSupportServiceInNetwork(guestNetworkId, Service.PortForwarding, provider)) { + pfRules.addAll(_pfRulesDao.listForApplication(ip.getId())); + } + if (_networkMgr.isProviderSupportServiceInNetwork(guestNetworkId, Service.StaticNat, provider)) { + staticNatFirewallRules.addAll(_rulesDao.listByIpAndPurpose(ip.getId(), Purpose.StaticNat)); + } + if (_networkMgr.isProviderSupportServiceInNetwork(guestNetworkId, Service.Firewall, provider)) { + firewallRules.addAll(_rulesDao.listByIpAndPurpose(ip.getId(), Purpose.Firewall)); + } + + if (_networkMgr.isProviderSupportServiceInNetwork(guestNetworkId, Service.Vpn, provider)) { + RemoteAccessVpn vpn = _vpnDao.findById(ip.getId()); + if (vpn != null) { + vpns.add(vpn); + } + } + + if (_networkMgr.isProviderSupportServiceInNetwork(guestNetworkId, Service.StaticNat, provider)) { + if (ip.isOneToOneNat()) { + String dstIp = _networkMgr.getIpInNetwork(ip.getAssociatedWithVmId(), guestNetworkId); + StaticNatImpl staticNat = new StaticNatImpl(ip.getAccountId(), ip.getDomainId(), guestNetworkId, ip.getId(), dstIp, false); + staticNats.add(staticNat); + } + } + } + + //Re-apply static nats + s_logger.debug("Found " + staticNats.size() + " static nat(s) to apply as a part of domR " + router + " start."); + if (!staticNats.isEmpty()) { + createApplyStaticNatCommands(staticNats, router, cmds, guestNetworkId); + } + + //Re-apply firewall rules + s_logger.debug("Found " + staticNats.size() + " firewall rule(s) to apply as a part of domR " + router + " start."); + if (!firewallRules.isEmpty()) { + createFirewallRulesCommands(firewallRules, router, cmds, guestNetworkId); + } + + // Re-apply port forwarding rules + s_logger.debug("Found " + pfRules.size() + " port forwarding rule(s) to apply as a part of domR " + router + " start."); + if (!pfRules.isEmpty()) { + createApplyPortForwardingRulesCommands(pfRules, router, cmds, guestNetworkId); + } + + // Re-apply static nat rules + s_logger.debug("Found " + staticNatFirewallRules.size() + " static nat rule(s) to apply as a part of domR " + router + " start."); + if (!staticNatFirewallRules.isEmpty()) { + List staticNatRules = new ArrayList(); + for (FirewallRule rule : staticNatFirewallRules) { + staticNatRules.add(_rulesMgr.buildStaticNatRule(rule, false)); + } + createApplyStaticNatRulesCommands(staticNatRules, router, cmds, guestNetworkId); + } + + // Re-apply vpn rules + s_logger.debug("Found " + vpns.size() + " vpn(s) to apply as a part of domR " + router + " start."); + if (!vpns.isEmpty()) { + for (RemoteAccessVpn vpn : vpns) { + createApplyVpnCommands(vpn, router, cmds); + } + } + + List lbs = _loadBalancerDao.listByNetworkId(guestNetworkId); + List lbRules = new ArrayList(); + if (_networkMgr.isProviderSupportServiceInNetwork(guestNetworkId, Service.Lb, provider)) { + // Re-apply load balancing rules + for (LoadBalancerVO lb : lbs) { + List dstList = _lbMgr.getExistingDestinations(lb.getId()); + List policyList = _lbMgr.getStickinessPolicies(lb.getId()); + LoadBalancingRule loadBalancing = new LoadBalancingRule(lb, dstList, policyList); + lbRules.add(loadBalancing); + } + } + + s_logger.debug("Found " + lbRules.size() + " load balancing rule(s) to apply as a part of domR " + router + " start."); + if (!lbRules.isEmpty()) { + createApplyLoadBalancingRulesCommands(lbRules, router, cmds, guestNetworkId); + } + } + } + + protected void finalizeIpAssocForNetwork(Commands cmds, VirtualRouter router, Provider provider, + Long guestNetworkId, Map vlanMacAddress) { + + ArrayList publicIps = getPublicIpsToApply(router, provider, guestNetworkId); + + if (publicIps != null && !publicIps.isEmpty()) { + s_logger.debug("Found " + publicIps.size() + " ip(s) to apply as a part of domR " + router + " start."); + // Re-apply public ip addresses - should come before PF/LB/VPN + if (_networkMgr.isProviderSupportServiceInNetwork(guestNetworkId, Service.Firewall, provider)) { + createAssociateIPCommands(router, publicIps, cmds, 0); + } + } + } + + protected ArrayList getPublicIpsToApply(VirtualRouter router, Provider provider, + Long guestNetworkId, com.cloud.network.IpAddress.State... skipInStates) { + long ownerId = router.getAccountId(); + final List userIps = _networkMgr.listPublicIpsAssignedToGuestNtwk(ownerId, guestNetworkId, null); + List allPublicIps = new ArrayList(); + if (userIps != null && !userIps.isEmpty()) { + boolean addIp = true; + for (IPAddressVO userIp : userIps) { + if (skipInStates != null) { + for (IpAddress.State stateToSkip : skipInStates) { + if (userIp.getState() == stateToSkip) { + s_logger.debug("Skipping ip address " + userIp + " in state " + userIp.getState()); + addIp = false; + break; + } + } + } + + if (addIp) { + PublicIp publicIp = new PublicIp(userIp, _vlanDao.findById(userIp.getVlanId()), + NetUtils.createSequenceBasedMacAddress(userIp.getMacAddress())); + allPublicIps.add(publicIp); + } + } + } + + //Get public Ips that should be handled by router + Network network = _networkDao.findById(guestNetworkId); + Map> ipToServices = _networkMgr.getIpToServices(allPublicIps, false, true); + Map> providerToIpList = _networkMgr.getProviderToIpList(network, ipToServices); + // Only cover virtual router for now, if ELB use it this need to be modified + + ArrayList publicIps = providerToIpList.get(provider); + return publicIps; + } + + @Override + public boolean finalizeStart(VirtualMachineProfile profile, long hostId, Commands cmds, + ReservationContext context) { + DomainRouterVO router = profile.getVirtualMachine(); + + boolean result = true; + + Answer answer = cmds.getAnswer("checkSsh"); + if (answer != null && answer instanceof CheckSshAnswer) { + CheckSshAnswer sshAnswer = (CheckSshAnswer) answer; + if (sshAnswer == null || !sshAnswer.getResult()) { + s_logger.warn("Unable to ssh to the VM: " + sshAnswer.getDetails()); + result = false; + } + } else { + result = false; + } + if (result == false) { + return result; + } + + //Get guest networks info + List guestNetworks = new ArrayList(); + + List routerNics = _nicDao.listByVmId(profile.getId()); + for (Nic routerNic : routerNics) { + Network network = _networkMgr.getNetwork(routerNic.getNetworkId()); + if (network.getTrafficType() == TrafficType.Guest) { + guestNetworks.add(network); + } + } + + answer = cmds.getAnswer("getDomRVersion"); + if (answer != null && answer instanceof GetDomRVersionAnswer) { + GetDomRVersionAnswer versionAnswer = (GetDomRVersionAnswer)answer; + if (answer == null || !answer.getResult()) { + s_logger.warn("Unable to get the template/scripts version of router " + router.getInstanceName() + + " due to: " + versionAnswer.getDetails()); + result = false; + } else { + router.setTemplateVersion(versionAnswer.getTemplateVersion()); + router.setScriptsVersion(versionAnswer.getScriptsVersion()); + router = _routerDao.persist(router, guestNetworks); + } + } else { + result = false; + } + + return result; + } + + @Override + public void finalizeStop(VirtualMachineProfile profile, StopAnswer answer) { + if (answer != null) { + VMInstanceVO vm = profile.getVirtualMachine(); + DomainRouterVO domR = _routerDao.findById(vm.getId()); + processStopOrRebootAnswer(domR, answer); + } + } + + @Override + public void finalizeExpunge(DomainRouterVO vm) { + } + + + @Override + public boolean startRemoteAccessVpn(Network network, RemoteAccessVpn vpn, List routers) + throws ResourceUnavailableException { + if (routers == null || routers.isEmpty()) { + s_logger.warn("Failed to start remote access VPN: no router found for account and zone"); + throw new ResourceUnavailableException("Failed to start remote access VPN: no router found for account and zone", + DataCenter.class, network.getDataCenterId()); + } + + for (VirtualRouter router : routers) { + if (router.getState() != State.Running) { + s_logger.warn("Failed to start remote access VPN: router not in right state " + router.getState()); + throw new ResourceUnavailableException("Failed to start remote access VPN: router not in right state " + + router.getState(), DataCenter.class, network.getDataCenterId()); + } + + Commands cmds = new Commands(OnError.Stop); + createApplyVpnCommands(vpn, router, cmds); + + try { + _agentMgr.send(router.getHostId(), cmds); + } catch (OperationTimedoutException e) { + s_logger.debug("Failed to start remote access VPN: ", e); + throw new AgentUnavailableException("Unable to send commands to virtual router ", router.getHostId(), e); + } + Answer answer = cmds.getAnswer("users"); + if (!answer.getResult()) { + s_logger.error("Unable to start vpn: unable add users to vpn in zone " + router.getDataCenterIdToDeployIn() + + " for account " + vpn.getAccountId() + " on domR: " + router.getInstanceName() + + " due to " + answer.getDetails()); + throw new ResourceUnavailableException("Unable to start vpn: Unable to add users to vpn in zone " + + router.getDataCenterIdToDeployIn() + " for account " + vpn.getAccountId() + " on domR: " + + router.getInstanceName() + " due to " + answer.getDetails(), DataCenter.class, router.getDataCenterIdToDeployIn()); + } + answer = cmds.getAnswer("startVpn"); + if (!answer.getResult()) { + s_logger.error("Unable to start vpn in zone " + router.getDataCenterIdToDeployIn() + " for account " + + vpn.getAccountId() + " on domR: " + router.getInstanceName() + " due to " + + answer.getDetails()); + throw new ResourceUnavailableException("Unable to start vpn in zone " + router.getDataCenterIdToDeployIn() + + " for account " + vpn.getAccountId() + " on domR: " + router.getInstanceName() + + " due to " + answer.getDetails(), DataCenter.class, router.getDataCenterIdToDeployIn()); + } + + } + return true; + } + + + @Override + public boolean deleteRemoteAccessVpn(Network network, RemoteAccessVpn vpn, List routers) + throws ResourceUnavailableException { + if (routers == null || routers.isEmpty()) { + s_logger.warn("Failed to delete remote access VPN: no router found for account and zone"); + throw new ResourceUnavailableException("Failed to delete remote access VPN", DataCenter.class, network.getDataCenterId()); + } + + boolean result = true; + for (VirtualRouter router : routers) { + if (router.getState() == State.Running) { + Commands cmds = new Commands(OnError.Continue); + IpAddress ip = _networkMgr.getIp(vpn.getServerAddressId()); + + RemoteAccessVpnCfgCommand removeVpnCmd = new RemoteAccessVpnCfgCommand(false, ip.getAddress().addr(), + vpn.getLocalIp(), vpn.getIpRange(), vpn.getIpsecPresharedKey()); + removeVpnCmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); + removeVpnCmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(network.getId(), router.getId())); + removeVpnCmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); + + DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); + removeVpnCmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); + + cmds.addCommand(removeVpnCmd); + + result = result && sendCommandsToRouter(router, cmds); + } else if (router.getState() == State.Stopped) { + s_logger.debug("Router " + router + " is in Stopped state, not sending deleteRemoteAccessVpn command to it"); + continue; + } else { + s_logger.warn("Failed to delete remote access VPN: domR " + router + " is not in right state " + router.getState()); + throw new ResourceUnavailableException("Failed to delete remote access VPN: domR is not in right state " + + router.getState(), DataCenter.class, network.getDataCenterId()); + } + } + + return result; + } + + + private DomainRouterVO start(DomainRouterVO router, User user, Account caller, Map params, DeploymentPlan planToDeploy) + throws StorageUnavailableException, InsufficientCapacityException, + ConcurrentOperationException, ResourceUnavailableException { + s_logger.debug("Starting router " + router); + if (_itMgr.start(router, params, user, caller, planToDeploy) != null) { + // We don't want the failure of VPN Connection affect the status of router, so we try to make connection only after router start successfully Long vpcId = router.getVpcId(); if (vpcId != null) { _s2sVpnMgr.reconnectDisconnectedVpnByVpc(vpcId); } - return _routerDao.findById(router.getId()); - } else { - return null; - } - } - - @Override - public DomainRouterVO stop(VirtualRouter router, boolean forced, User user, Account caller) throws ConcurrentOperationException, ResourceUnavailableException { - s_logger.debug("Stopping router " + router); - try { - if (_itMgr.advanceStop((DomainRouterVO) router, forced, user, caller)) { - return _routerDao.findById(router.getId()); - } else { - return null; - } - } catch (OperationTimedoutException e) { - throw new CloudRuntimeException("Unable to stop " + router, e); - } - } - - @Override - public boolean applyDhcpEntry(Network network, final NicProfile nic, VirtualMachineProfile profile, - DeployDestination dest, List routers) - throws ResourceUnavailableException { - _userVmDao.loadDetails((UserVmVO) profile.getVirtualMachine()); - - final VirtualMachineProfile updatedProfile = profile; - final boolean isZoneBasic = (dest.getDataCenter().getNetworkType() == NetworkType.Basic); - final Long podId = isZoneBasic ? dest.getPod().getId() : null; - - boolean podLevelException = false; - //for user vm in Basic zone we should try to re-deploy vm in a diff pod if it fails to deploy in original pod; so throwing exception with Pod scope - if (isZoneBasic && podId != null && updatedProfile.getVirtualMachine().getType() == VirtualMachine.Type.User - && network.getTrafficType() == TrafficType.Guest && network.getGuestType() == Network.GuestType.Shared) { - podLevelException = true; - } - - return applyRules(network, routers, "dhcp entry", podLevelException, podId, true, new RuleApplier() { - @Override - public boolean execute(Network network, VirtualRouter router) throws ResourceUnavailableException { - //for basic zone, send dhcp/dns information to all routers in the basic network only when _dnsBasicZoneUpdates is set to "all" value - Commands cmds = new Commands(OnError.Stop); - if (!(isZoneBasic && router.getPodIdToDeployIn().longValue() != podId.longValue() && _dnsBasicZoneUpdates.equalsIgnoreCase("pod"))) { - NicVO nicVo = _nicDao.findById(nic.getId()); - createDhcpEntryCommand(router, updatedProfile.getVirtualMachine(), nicVo, cmds); - return sendCommandsToRouter(router, cmds); - } - return true; - } - }); - } - - private String findDefaultDnsIp(long userVmId) { - NicVO defaultNic = _nicDao.findDefaultNicForVM(userVmId); - - //check if DNS provider is the domR - if (!_networkMgr.isProviderSupportServiceInNetwork(defaultNic.getNetworkId(), Service.Dns, Provider.VirtualRouter)) { - return null; - } - - NetworkOfferingVO offering = _networkOfferingDao.findById(_networkDao.findById(defaultNic.getNetworkId()).getNetworkOfferingId()); - if (offering.getRedundantRouter()) { - return findGatewayIp(userVmId); - } - - //find domR's nic in the network - NicVO domrDefaultNic = _nicDao.findByNetworkIdAndType(defaultNic.getNetworkId(), VirtualMachine.Type.DomainRouter); - return domrDefaultNic.getIp4Address(); - } - - private String findGatewayIp(long userVmId) { - NicVO defaultNic = _nicDao.findDefaultNicForVM(userVmId); - return defaultNic.getGateway(); - } - - @Override - public boolean applyUserData(Network network, final NicProfile nic, VirtualMachineProfile profile, DeployDestination dest, List routers) - throws ResourceUnavailableException { - _userVmDao.loadDetails((UserVmVO) profile.getVirtualMachine()); - - final VirtualMachineProfile updatedProfile = profile; - final boolean isZoneBasic = (dest.getDataCenter().getNetworkType() == NetworkType.Basic); - final Long podId = isZoneBasic ? dest.getPod().getId() : null; - - boolean podLevelException = false; - //for user vm in Basic zone we should try to re-deploy vm in a diff pod if it fails to deploy in original pod; so throwing exception with Pod scope - if (isZoneBasic && podId != null && updatedProfile.getVirtualMachine().getType() == VirtualMachine.Type.User - && network.getTrafficType() == TrafficType.Guest && network.getGuestType() == Network.GuestType.Shared) { - podLevelException = true; - } - - return applyRules(network, routers, "userdata and password entry", podLevelException, podId, false, new RuleApplier() { - @Override - public boolean execute(Network network, VirtualRouter router) throws ResourceUnavailableException { - //for basic zone, send vm data/password information only to the router in the same pod - Commands cmds = new Commands(OnError.Stop); - if (!(isZoneBasic && router.getPodIdToDeployIn().longValue() != podId.longValue())) { - NicVO nicVo = _nicDao.findById(nic.getId()); - createPasswordCommand(router, updatedProfile, nicVo, cmds); - createVmDataCommand(router, updatedProfile.getVirtualMachine(), nicVo, updatedProfile.getVirtualMachine().getDetail("SSH.PublicKey"), cmds); - return sendCommandsToRouter(router, cmds); - } - return true; - } - }); - } - - @Override - public DomainRouterVO persist(DomainRouterVO router) { - DomainRouterVO virtualRouter = _routerDao.persist(router); - return virtualRouter; - } - - @Override - //FIXME add partial success and STOP state support - public String[] applyVpnUsers(Network network, List users, List routers) throws ResourceUnavailableException { - if (routers == null || routers.isEmpty()) { - s_logger.warn("Failed to add/remove VPN users: no router found for account and zone"); - throw new ResourceUnavailableException("Unable to assign ip addresses, domR doesn't exist for network " + - network.getId(), DataCenter.class, network.getDataCenterId()); - } - - boolean agentResults = true; - - for (DomainRouterVO router : routers) { - if (router.getState() != State.Running) { - s_logger.warn("Failed to add/remove VPN users: router not in running state"); - throw new ResourceUnavailableException("Unable to assign ip addresses, domR is not in right state " + - router.getState(), DataCenter.class, network.getDataCenterId()); - } - - Commands cmds = new Commands(OnError.Continue); - List addUsers = new ArrayList(); - List removeUsers = new ArrayList(); - for (VpnUser user : users) { - if (user.getState() == VpnUser.State.Add || user.getState() == VpnUser.State.Active) { - addUsers.add(user); - } else if (user.getState() == VpnUser.State.Revoke) { - removeUsers.add(user); - } - } - - VpnUsersCfgCommand cmd = new VpnUsersCfgCommand(addUsers, removeUsers); - cmd.setAccessDetail(NetworkElementCommand.ACCOUNT_ID, String.valueOf(router.getAccountId())); - cmd.setAccessDetail(NetworkElementCommand.GUEST_NETWORK_CIDR, network.getCidr()); - cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); - cmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(network.getId(), router.getId())); - cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); - DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); - cmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); - - cmds.addCommand(cmd); - - - // Currently we receive just one answer from the agent. In the future we have to parse individual answers and set - // results accordingly - boolean agentResult = sendCommandsToRouter(router, cmds); - agentResults = agentResults && agentResult; - } - - String[] result = new String[users.size()]; - for (int i = 0; i < result.length; i++) { - if (agentResults) { - result[i] = null; - } else { - result[i] = String.valueOf(agentResults); - } - } - - return result; - } - - @Override - public DomainRouterVO findById(long id) { - return _routerDao.findById(id); - } - - @Override - public DomainRouterVO findByName(String name) { - if (!VirtualMachineName.isValidRouterName(name)) { - return null; - } - - return _routerDao.findById(VirtualMachineName.getRouterId(name)); - } - - @Override @ActionEvent(eventType = EventTypes.EVENT_ROUTER_START, eventDescription = "starting router Vm", async = true) - public VirtualRouter startRouter(long id) throws ResourceUnavailableException, InsufficientCapacityException, ConcurrentOperationException{ - return startRouter(id, true); - } - - @Override - public VirtualRouter startRouter(long routerId, boolean reprogramNetwork) throws ResourceUnavailableException, - InsufficientCapacityException, ConcurrentOperationException { - Account caller = UserContext.current().getCaller(); - User callerUser = _accountMgr.getActiveUser(UserContext.current().getCallerUserId()); - - // verify parameters - DomainRouterVO router = _routerDao.findById(routerId); - if (router == null) { - throw new InvalidParameterValueException("Unable to find router by id " + routerId + "."); - } - _accountMgr.checkAccess(caller, null, true, router); - - Account owner = _accountMgr.getAccount(router.getAccountId()); - - // Check if all networks are implemented for the domR; if not - implement them - DataCenter dc = _dcDao.findById(router.getDataCenterIdToDeployIn()); - HostPodVO pod = null; - if (router.getPodIdToDeployIn() != null) { - pod = _podDao.findById(router.getPodIdToDeployIn()); - } - DeployDestination dest = new DeployDestination(dc, pod, null, null); - - ReservationContext context = new ReservationContextImpl(null, null, callerUser, owner); - - List nics = _nicDao.listByVmId(routerId); - - for (NicVO nic : nics) { - if (!_networkMgr.startNetwork(nic.getNetworkId(), dest, context)) { - s_logger.warn("Failed to start network id=" + nic.getNetworkId() + " as a part of domR start"); - throw new CloudRuntimeException("Failed to start network id=" + nic.getNetworkId() + " as a part of domR start"); - } - } - - UserVO user = _userDao.findById(UserContext.current().getCallerUserId()); - Map params = new HashMap(); - if (reprogramNetwork) { - params.put(Param.ReProgramGuestNetworks, true); - } else { - params.put(Param.ReProgramGuestNetworks, false); - } - VirtualRouter virtualRouter = startVirtualRouter(router, user, caller, params); - if(virtualRouter == null){ - throw new CloudRuntimeException("Failed to start router with id " + routerId); - } - return virtualRouter; - } - - private void createAssociateIPCommands(final VirtualRouter router, final List ips, Commands cmds, long vmId) { - - // Ensure that in multiple vlans case we first send all ip addresses of vlan1, then all ip addresses of vlan2, etc.. - Map> vlanIpMap = new HashMap>(); - for (final PublicIpAddress ipAddress : ips) { - String vlanTag = ipAddress.getVlanTag(); - ArrayList ipList = vlanIpMap.get(vlanTag); - if (ipList == null) { - ipList = new ArrayList(); - } - //domR doesn't support release for sourceNat IP address; so reset the state - if (ipAddress.isSourceNat() && ipAddress.getState() == IpAddress.State.Releasing) { - ipAddress.setState(IpAddress.State.Allocated); - } - ipList.add(ipAddress); - vlanIpMap.put(vlanTag, ipList); - } - - for (Map.Entry> vlanAndIp : vlanIpMap.entrySet()) { - List ipAddrList = vlanAndIp.getValue(); - // Source nat ip address should always be sent first - Collections.sort(ipAddrList, new Comparator() { - @Override - public int compare(PublicIpAddress o1, PublicIpAddress o2) { - boolean s1 = o1.isSourceNat(); - boolean s2 = o2.isSourceNat(); - return (s1 ^ s2) ? ((s1 ^ true) ? 1 : -1) : 0; - } - }); - - // Get network rate - required for IpAssoc - Integer networkRate = _networkMgr.getNetworkRate(ipAddrList.get(0).getNetworkId(), router.getId()); - Network network = _networkMgr.getNetwork(ipAddrList.get(0).getNetworkId()); - - IpAddressTO[] ipsToSend = new IpAddressTO[ipAddrList.size()]; - int i = 0; - boolean firstIP = true; - - for (final PublicIpAddress ipAddr : ipAddrList) { - - boolean add = (ipAddr.getState() == IpAddress.State.Releasing ? false : true); - boolean sourceNat = ipAddr.isSourceNat(); - /* enable sourceNAT for the first ip of the public interface */ - if (firstIP) { - sourceNat = true; - } - String vlanId = ipAddr.getVlanTag(); - String vlanGateway = ipAddr.getGateway(); - String vlanNetmask = ipAddr.getNetmask(); - String vifMacAddress = ipAddr.getMacAddress(); - - String vmGuestAddress = null; - - IpAddressTO ip = new IpAddressTO(ipAddr.getAccountId(), ipAddr.getAddress().addr(), add, firstIP, - sourceNat, vlanId, vlanGateway, vlanNetmask, vifMacAddress, vmGuestAddress, networkRate, ipAddr.isOneToOneNat()); - - ip.setTrafficType(network.getTrafficType()); - ip.setNetworkName(_networkMgr.getNetworkTag(router.getHypervisorType(), network)); - ipsToSend[i++] = ip; - /* send the firstIP = true for the first Add, this is to create primary on interface*/ - if (!firstIP || add) { - firstIP = false; - } - } - IpAssocCommand cmd = new IpAssocCommand(ipsToSend); - cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); - cmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(ipAddrList.get(0).getNetworkId(), router.getId())); - cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); - DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); - cmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); - - cmds.addCommand("IPAssocCommand", cmd); - } - } - - private void createApplyPortForwardingRulesCommands(List rules, VirtualRouter router, Commands cmds, long guestNetworkId) { - List rulesTO = null; - if (rules != null) { - rulesTO = new ArrayList(); - for (PortForwardingRule rule : rules) { - IpAddress sourceIp = _networkMgr.getIp(rule.getSourceIpAddressId()); - PortForwardingRuleTO ruleTO = new PortForwardingRuleTO(rule, null, sourceIp.getAddress().addr()); - rulesTO.add(ruleTO); - } - } - - SetPortForwardingRulesCommand cmd = null; - - if (router.getVpcId() != null) { - cmd = new SetPortForwardingRulesVpcCommand(rulesTO); - } else { - cmd = new SetPortForwardingRulesCommand(rulesTO); - } - - cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); - cmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(guestNetworkId, router.getId())); - cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); - DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); - cmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); - - cmds.addCommand(cmd); - } - - private void createApplyStaticNatRulesCommands(List rules, VirtualRouter router, Commands cmds, long guestNetworkId) { - List rulesTO = null; - if (rules != null) { - rulesTO = new ArrayList(); - for (StaticNatRule rule : rules) { - IpAddress sourceIp = _networkMgr.getIp(rule.getSourceIpAddressId()); - StaticNatRuleTO ruleTO = new StaticNatRuleTO(rule, null, sourceIp.getAddress().addr(), rule.getDestIpAddress()); - rulesTO.add(ruleTO); - } - } - - SetStaticNatRulesCommand cmd = new SetStaticNatRulesCommand(rulesTO, router.getVpcId()); - cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); - cmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(guestNetworkId, router.getId())); - cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); - DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); - cmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); - cmds.addCommand(cmd); - } - - private void createApplyLoadBalancingRulesCommands(List rules, VirtualRouter router, Commands cmds, long guestNetworkId) { - - LoadBalancerTO[] lbs = new LoadBalancerTO[rules.size()]; - int i = 0; - for (LoadBalancingRule rule : rules) { - boolean revoked = (rule.getState().equals(FirewallRule.State.Revoke)); - String protocol = rule.getProtocol(); - String algorithm = rule.getAlgorithm(); - - String srcIp = _networkMgr.getIp(rule.getSourceIpAddressId()).getAddress().addr(); - int srcPort = rule.getSourcePortStart(); - List destinations = rule.getDestinations(); - List stickinessPolicies = rule.getStickinessPolicies(); - LoadBalancerTO lb = new LoadBalancerTO(srcIp, srcPort, protocol, algorithm, revoked, false, destinations, stickinessPolicies); - lbs[i++] = lb; - } - String routerPublicIp = null; - - if (router instanceof DomainRouterVO) { - DomainRouterVO domr = _routerDao.findById(router.getId()); - routerPublicIp = domr.getPublicIpAddress(); - } - - Network guestNetwork = _networkMgr.getNetwork(guestNetworkId); - Nic nic = _nicDao.findByInstanceIdAndNetworkId(guestNetwork.getId(), router.getId()); - NicProfile nicProfile = new NicProfile(nic, guestNetwork, nic.getBroadcastUri(), nic.getIsolationUri(), - _networkMgr.getNetworkRate(guestNetwork.getId(), router.getId()), - _networkMgr.isSecurityGroupSupportedInNetwork(guestNetwork), - _networkMgr.getNetworkTag(router.getHypervisorType(), guestNetwork)); - - LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(lbs,routerPublicIp, - getRouterIpInNetwork(guestNetworkId, router.getId()),router.getPrivateIpAddress(), - _itMgr.toNicTO(nicProfile, router.getHypervisorType()), router.getVpcId()); - - cmd.lbStatsVisibility = _configDao.getValue(Config.NetworkLBHaproxyStatsVisbility.key()); - cmd.lbStatsUri = _configDao.getValue(Config.NetworkLBHaproxyStatsUri.key()); - cmd.lbStatsAuth = _configDao.getValue(Config.NetworkLBHaproxyStatsAuth.key()); - cmd.lbStatsPort = _configDao.getValue(Config.NetworkLBHaproxyStatsPort.key()); - - - cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); - cmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(guestNetworkId, router.getId())); - cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); - DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); - cmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); - cmds.addCommand(cmd); - - } - - private void createApplyVpnCommands(RemoteAccessVpn vpn, VirtualRouter router, Commands cmds) { - List vpnUsers = _vpnUsersDao.listByAccount(vpn.getAccountId()); - List addUsers = new ArrayList(); - List removeUsers = new ArrayList(); - for (VpnUser user : vpnUsers) { - if (user.getState() == VpnUser.State.Add) { - addUsers.add(user); - } else if (user.getState() == VpnUser.State.Revoke) { - removeUsers.add(user); - } - } - - VpnUsersCfgCommand addUsersCmd = new VpnUsersCfgCommand(addUsers, removeUsers); - addUsersCmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); - addUsersCmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(vpn.getNetworkId(), router.getId())); - addUsersCmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); - - IpAddress ip = _networkMgr.getIp(vpn.getServerAddressId()); - - RemoteAccessVpnCfgCommand startVpnCmd = new RemoteAccessVpnCfgCommand(true, ip.getAddress().addr(), - vpn.getLocalIp(), vpn.getIpRange(), vpn.getIpsecPresharedKey()); - startVpnCmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); - startVpnCmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(vpn.getNetworkId(), router.getId())); - startVpnCmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); - DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); - startVpnCmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); - - cmds.addCommand("users", addUsersCmd); - cmds.addCommand("startVpn", startVpnCmd); - } - - private void createPasswordCommand(VirtualRouter router, VirtualMachineProfile profile, NicVO nic, Commands cmds) { - String password = (String) profile.getParameter(VirtualMachineProfile.Param.VmPassword); - DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); - - // password should be set only on default network element - if (password != null && nic.isDefaultNic()) { - final String encodedPassword = PasswordGenerator.rot13(password); - SavePasswordCommand cmd = new SavePasswordCommand(encodedPassword, nic.getIp4Address(), profile.getVirtualMachine().getHostName()); - cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); - cmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(nic.getNetworkId(), router.getId())); - cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); - cmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); - - cmds.addCommand("password", cmd); - } - - } - - private void createVmDataCommand(VirtualRouter router, UserVm vm, NicVO nic, String publicKey, Commands cmds) { - String serviceOffering = _serviceOfferingDao.findByIdIncludingRemoved(vm.getServiceOfferingId()).getDisplayText(); - String zoneName = _dcDao.findById(router.getDataCenterIdToDeployIn()).getName(); - cmds.addCommand("vmdata", - generateVmDataCommand(router, nic.getIp4Address(), vm.getUserData(), serviceOffering, zoneName, nic.getIp4Address(), - vm.getHostName(), vm.getInstanceName(), vm.getId(), publicKey, nic.getNetworkId())); - - } - - private void createVmDataCommandForVMs(DomainRouterVO router, Commands cmds, long guestNetworkId) { - List vms = _userVmDao.listByNetworkIdAndStates(guestNetworkId, State.Running, State.Migrating, State.Stopping); - DataCenterVO dc = _dcDao.findById(router.getDataCenterIdToDeployIn()); - for (UserVmVO vm : vms) { - boolean createVmData = true; - if (dc.getNetworkType() == NetworkType.Basic && router.getPodIdToDeployIn().longValue() != vm.getPodIdToDeployIn().longValue()) { - createVmData = false; - } - - if (createVmData) { - NicVO nic = _nicDao.findByInstanceIdAndNetworkId(guestNetworkId, vm.getId()); - if (nic != null) { - s_logger.debug("Creating user data entry for vm " + vm + " on domR " + router); - createVmDataCommand(router, vm, nic, null, cmds); - } - } - } - } - - private void createDhcpEntryCommand(VirtualRouter router, UserVm vm, NicVO nic, Commands cmds) { - DhcpEntryCommand dhcpCommand = new DhcpEntryCommand(nic.getMacAddress(), nic.getIp4Address(), vm.getHostName()); - DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); - String gatewayIp = findGatewayIp(vm.getId()); - if (!gatewayIp.equals(nic.getGateway())) { - GuestOSVO guestOS = _guestOSDao.findById(vm.getGuestOSId()); - // Don't set dhcp:router option for non-default nic on CentOS/RHEL, because they would set routing on wrong interface - // This is tricky, we may need to update this when we have more information on various OS's behavior - if (guestOS.getDisplayName().startsWith("CentOS") || guestOS.getDisplayName().startsWith("Red Hat Enterprise")) { - gatewayIp = "0.0.0.0"; - } - } - dhcpCommand.setDefaultRouter(gatewayIp); - dhcpCommand.setDefaultDns(findDefaultDnsIp(vm.getId())); - - dhcpCommand.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); - dhcpCommand.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); - dhcpCommand.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(nic.getNetworkId(), router.getId())); - dhcpCommand.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); - - cmds.addCommand("dhcp", dhcpCommand); - } - - private void createDhcpEntryCommandsForVMs(DomainRouterVO router, Commands cmds, long guestNetworkId) { - List vms = _userVmDao.listByNetworkIdAndStates(guestNetworkId, State.Running, State.Migrating, State.Stopping); - DataCenterVO dc = _dcDao.findById(router.getDataCenterIdToDeployIn()); - for (UserVmVO vm : vms) { - boolean createDhcp = true; - if (dc.getNetworkType() == NetworkType.Basic && router.getPodIdToDeployIn().longValue() != vm.getPodIdToDeployIn().longValue() - && _dnsBasicZoneUpdates.equalsIgnoreCase("pod")) { - createDhcp = false; - } - if (createDhcp) { - NicVO nic = _nicDao.findByInstanceIdAndNetworkId(guestNetworkId, vm.getId()); - if (nic != null) { - s_logger.debug("Creating dhcp entry for vm " + vm + " on domR " + router + "."); - createDhcpEntryCommand(router, vm, nic, cmds); - } - } - } - } - - protected boolean sendCommandsToRouter(final VirtualRouter router, Commands cmds) throws AgentUnavailableException { - Answer[] answers = null; - try { - answers = _agentMgr.send(router.getHostId(), cmds); - } catch (OperationTimedoutException e) { - s_logger.warn("Timed Out", e); - throw new AgentUnavailableException("Unable to send commands to virtual router ", router.getHostId(), e); - } - - if (answers == null) { - return false; - } - - if (answers.length != cmds.size()) { - return false; - } - - // FIXME: Have to return state for individual command in the future - boolean result = true; - if (answers.length > 0) { - for (Answer answer : answers) { - if (!answer.getResult()) { - result = false; - break; - } - } - } - return result; - } - - protected void handleSingleWorkingRedundantRouter(List connectedRouters, List disconnectedRouters, String reason) throws ResourceUnavailableException - { - if (connectedRouters.isEmpty() || disconnectedRouters.isEmpty()) { - return; - } - if (connectedRouters.size() != 1 || disconnectedRouters.size() != 1) { - s_logger.warn("How many redundant routers do we have?? "); - return; - } - if (!connectedRouters.get(0).getIsRedundantRouter()) { - throw new ResourceUnavailableException("Who is calling this with non-redundant router or non-domain router?", - DataCenter.class, connectedRouters.get(0).getDataCenterIdToDeployIn()); - } - if (!disconnectedRouters.get(0).getIsRedundantRouter()) { - throw new ResourceUnavailableException("Who is calling this with non-redundant router or non-domain router?", - DataCenter.class, disconnectedRouters.get(0).getDataCenterIdToDeployIn()); - } - - DomainRouterVO connectedRouter = (DomainRouterVO)connectedRouters.get(0); - DomainRouterVO disconnectedRouter = (DomainRouterVO)disconnectedRouters.get(0); - - if (s_logger.isDebugEnabled()) { - s_logger.debug("About to stop the router " + disconnectedRouter.getInstanceName() + " due to: " + reason); - } - String title = "Virtual router " + disconnectedRouter.getInstanceName() + " would be stopped after connecting back, due to " + reason; - String context = "Virtual router (name: " + disconnectedRouter.getInstanceName() + ", id: " + disconnectedRouter.getId() + ") would be stopped after connecting back, due to: " + reason; - _alertMgr.sendAlert(AlertManager.ALERT_TYPE_DOMAIN_ROUTER, - disconnectedRouter.getDataCenterIdToDeployIn(), disconnectedRouter.getPodIdToDeployIn(), title, context); - disconnectedRouter.setStopPending(true); - disconnectedRouter = _routerDao.persist(disconnectedRouter); - - int connRouterPR = getRealPriority(connectedRouter); - int disconnRouterPR = getRealPriority(disconnectedRouter); - if (connRouterPR < disconnRouterPR) { - //connRouterPR < disconnRouterPR, they won't equal at anytime - if (!connectedRouter.getIsPriorityBumpUp()) { - final BumpUpPriorityCommand command = new BumpUpPriorityCommand(); - command.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(connectedRouter.getId())); - command.setAccessDetail(NetworkElementCommand.ROUTER_NAME, connectedRouter.getInstanceName()); - final Answer answer = _agentMgr.easySend(connectedRouter.getHostId(), command); - if (!answer.getResult()) { - s_logger.error("Failed to bump up " + connectedRouter.getInstanceName() + "'s priority! " + answer.getDetails()); - } - } else { - String t = "Can't bump up virtual router " + connectedRouter.getInstanceName() + "'s priority due to it's already bumped up!"; - _alertMgr.sendAlert(AlertManager.ALERT_TYPE_DOMAIN_ROUTER, - connectedRouter.getDataCenterIdToDeployIn(), connectedRouter.getPodIdToDeployIn(), t, t); - } - } - } - - @Override - public boolean associatePublicIP(Network network, final List ipAddress, List routers) - throws ResourceUnavailableException { - if (ipAddress == null || ipAddress.isEmpty()) { - s_logger.debug("No ip association rules to be applied for network " + network.getId()); - return true; - } - return applyRules(network, routers, "ip association", false, null, false, new RuleApplier() { - @Override - public boolean execute(Network network, VirtualRouter router) throws ResourceUnavailableException { - Commands cmds = new Commands(OnError.Continue); - createAssociateIPCommands(router, ipAddress, cmds, 0); - return sendCommandsToRouter(router, cmds); - } - }); - } - - @Override - public boolean applyFirewallRules(Network network, final List rules, List routers) throws ResourceUnavailableException { - if (rules == null || rules.isEmpty()) { - s_logger.debug("No firewall rules to be applied for network " + network.getId()); - return true; - } - return applyRules(network, routers, "firewall rules", false, null, false, new RuleApplier() { - @Override - public boolean execute(Network network, VirtualRouter router) throws ResourceUnavailableException { - if (rules.get(0).getPurpose() == Purpose.LoadBalancing) { - // for load balancer we have to resend all lb rules for the network - List lbs = _loadBalancerDao.listByNetworkId(network.getId()); - List lbRules = new ArrayList(); - for (LoadBalancerVO lb : lbs) { - List dstList = _lbMgr.getExistingDestinations(lb.getId()); - List policyList = _lbMgr.getStickinessPolicies(lb.getId()); - LoadBalancingRule loadBalancing = new LoadBalancingRule(lb, dstList,policyList); - lbRules.add(loadBalancing); - } - return sendLBRules(router, lbRules, network.getId()); - } else if (rules.get(0).getPurpose() == Purpose.PortForwarding) { - return sendPortForwardingRules(router, (List) rules, network.getId()); - } else if (rules.get(0).getPurpose() == Purpose.StaticNat) { - return sendStaticNatRules(router, (List) rules, network.getId()); - } else if (rules.get(0).getPurpose() == Purpose.Firewall) { - return sendFirewallRules(router, (List) rules, network.getId()); - } else { - s_logger.warn("Unable to apply rules of purpose: " + rules.get(0).getPurpose()); - return false; - } - } - }); - } - - protected boolean sendLBRules(VirtualRouter router, List rules, long guestNetworkId) throws ResourceUnavailableException { - Commands cmds = new Commands(OnError.Continue); - createApplyLoadBalancingRulesCommands(rules, router, cmds, guestNetworkId); - return sendCommandsToRouter(router, cmds); - } - - protected boolean sendPortForwardingRules(VirtualRouter router, List rules, long guestNetworkId) throws ResourceUnavailableException { - Commands cmds = new Commands(OnError.Continue); - createApplyPortForwardingRulesCommands(rules, router, cmds, guestNetworkId); - return sendCommandsToRouter(router, cmds); - } - - protected boolean sendStaticNatRules(VirtualRouter router, List rules, long guestNetworkId) throws ResourceUnavailableException { - Commands cmds = new Commands(OnError.Continue); - createApplyStaticNatRulesCommands(rules, router, cmds, guestNetworkId); - return sendCommandsToRouter(router, cmds); - } - - @Override - public List getRoutersForNetwork(long networkId) { - List routers = _routerDao.findByNetwork(networkId); - List vrs = new ArrayList(routers.size()); - for (DomainRouterVO router : routers) { - vrs.add(router); - } - return vrs; - } - - private void createFirewallRulesCommands(List rules, VirtualRouter router, Commands cmds, long guestNetworkId) { - List rulesTO = null; - if (rules != null) { - rulesTO = new ArrayList(); - for (FirewallRule rule : rules) { - IpAddress sourceIp = _networkMgr.getIp(rule.getSourceIpAddressId()); - FirewallRuleTO ruleTO = new FirewallRuleTO(rule, null, sourceIp.getAddress().addr()); - rulesTO.add(ruleTO); - } - } - - SetFirewallRulesCommand cmd = new SetFirewallRulesCommand(rulesTO); - cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); - cmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(guestNetworkId, router.getId())); - cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); - DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); - cmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); - cmds.addCommand(cmd); - } - - - protected boolean sendFirewallRules(VirtualRouter router, List rules, long guestNetworkId) throws ResourceUnavailableException { - Commands cmds = new Commands(OnError.Continue); - createFirewallRulesCommands(rules, router, cmds, guestNetworkId); - return sendCommandsToRouter(router, cmds); - } - - @Override - public String getDnsBasicZoneUpdate() { - return _dnsBasicZoneUpdates; - } - - protected interface RuleApplier { - boolean execute(Network network, VirtualRouter router) throws ResourceUnavailableException; - } - - protected boolean applyRules(Network network, List routers, String typeString, - boolean isPodLevelException, Long podId, boolean failWhenDisconnect, RuleApplier applier) throws ResourceUnavailableException { - if (routers == null || routers.isEmpty()) { - s_logger.warn("Unable to apply " + typeString + ", virtual router doesn't exist in the network " + network.getId()); - throw new ResourceUnavailableException("Unable to apply " + typeString , DataCenter.class, network.getDataCenterId()); - } - - DataCenter dc = _dcDao.findById(network.getDataCenterId()); - boolean isZoneBasic = (dc.getNetworkType() == NetworkType.Basic); - - // isPodLevelException and podId is only used for basic zone - assert !((!isZoneBasic && isPodLevelException) || (isZoneBasic && isPodLevelException && podId == null)); - - List connectedRouters = new ArrayList(); - List disconnectedRouters = new ArrayList(); - boolean result = true; - String msg = "Unable to apply " + typeString + " on disconnected router "; - for (VirtualRouter router : routers) { - if (router.getState() == State.Running) { - s_logger.debug("Applying " + typeString + " in network " + network); - - if (router.isStopPending()) { - if (_hostDao.findById(router.getHostId()).getStatus() == Status.Up) { - throw new ResourceUnavailableException("Unable to process due to the stop pending router " + - router.getInstanceName() + " haven't been stopped after it's host coming back!", - DataCenter.class, router.getDataCenterIdToDeployIn()); - } - s_logger.debug("Router " + router.getInstanceName() + " is stop pending, so not sending apply " + - typeString + " commands to the backend"); - continue; - } - try { - result = applier.execute(network, router); - connectedRouters.add(router); - } catch (AgentUnavailableException e) { - s_logger.warn(msg + router.getInstanceName(), e); - disconnectedRouters.add(router); - } - - //If rules fail to apply on one domR and not due to disconnection, no need to proceed with the rest - if (!result) { - if (isZoneBasic && isPodLevelException) { - throw new ResourceUnavailableException("Unable to apply " + typeString + " on router ", Pod.class, podId); - } - throw new ResourceUnavailableException("Unable to apply " + typeString + " on router ", DataCenter.class, - router.getDataCenterIdToDeployIn()); - } - - } else if (router.getState() == State.Stopped || router.getState() == State.Stopping) { - s_logger.debug("Router " + router.getInstanceName() + " is in " + router.getState() + - ", so not sending apply " + typeString + " commands to the backend"); - } else { - s_logger.warn("Unable to apply " + typeString +", virtual router is not in the right state " + router.getState()); - if (isZoneBasic && isPodLevelException) { - throw new ResourceUnavailableException("Unable to apply " + typeString + - ", virtual router is not in the right state", Pod.class, podId); - } - throw new ResourceUnavailableException("Unable to apply " + typeString + - ", virtual router is not in the right state", DataCenter.class, router.getDataCenterIdToDeployIn()); - } - } - - if (!connectedRouters.isEmpty()) { - if (!isZoneBasic && !disconnectedRouters.isEmpty() && disconnectedRouters.get(0).getIsRedundantRouter()) { - // These disconnected redundant virtual routers are out of sync now, stop them for synchronization - handleSingleWorkingRedundantRouter(connectedRouters, disconnectedRouters, msg); - } - } else if (!disconnectedRouters.isEmpty()) { - for (VirtualRouter router : disconnectedRouters) { - if (s_logger.isDebugEnabled()) { - s_logger.debug(msg + router.getInstanceName() + "(" + router.getId() + ")"); - } - } - if (isZoneBasic && isPodLevelException) { - throw new ResourceUnavailableException(msg, Pod.class, podId); - } - throw new ResourceUnavailableException(msg, DataCenter.class, disconnectedRouters.get(0).getDataCenterIdToDeployIn()); - } - - result = true; - if (failWhenDisconnect) { - result = !connectedRouters.isEmpty(); - } - return result; - } - - @Override - public boolean applyStaticNats(Network network, final List rules, List routers) throws ResourceUnavailableException { - if (rules == null || rules.isEmpty()) { - s_logger.debug("No static nat rules to be applied for network " + network.getId()); - return true; - } - return applyRules(network, routers, "static nat rules", false, null, false, new RuleApplier() { - @Override - public boolean execute(Network network, VirtualRouter router) throws ResourceUnavailableException { - return applyStaticNat(router, rules, network.getId()); - } - }); - } - - - protected boolean applyStaticNat(VirtualRouter router, List rules, long guestNetworkId) throws ResourceUnavailableException { - Commands cmds = new Commands(OnError.Continue); - createApplyStaticNatCommands(rules, router, cmds, guestNetworkId); - return sendCommandsToRouter(router, cmds); - } - - private void createApplyStaticNatCommands(List rules, VirtualRouter router, Commands cmds, - long guestNetworkId) { - List rulesTO = null; - if (rules != null) { - rulesTO = new ArrayList(); - for (StaticNat rule : rules) { - IpAddress sourceIp = _networkMgr.getIp(rule.getSourceIpAddressId()); - StaticNatRuleTO ruleTO = new StaticNatRuleTO(0, sourceIp.getAddress().addr(), null, - null, rule.getDestIpAddress(), null, null, null, rule.isForRevoke(), false); - rulesTO.add(ruleTO); - } - } - - SetStaticNatRulesCommand cmd = new SetStaticNatRulesCommand(rulesTO, router.getVpcId()); - cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); - cmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(guestNetworkId, router.getId())); - cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); - DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); - cmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); - cmds.addCommand(cmd); - } - - @Override - public int getTimeout() { - return -1; - } - - @Override - public boolean isRecurring() { - return false; - } - - @Override - public boolean processAnswers(long agentId, long seq, Answer[] answers) { - return false; - } - - @Override - public boolean processCommands(long agentId, long seq, Command[] commands) { - return false; - } - - @Override - public void processConnect(HostVO host, StartupCommand cmd, boolean forRebalance) throws ConnectionException { - UserContext context = UserContext.current(); - context.setAccountId(1); - List routers = _routerDao.listIsolatedByHostId(host.getId()); - for (DomainRouterVO router : routers) { - if (router.isStopPending()) { - State state = router.getState(); - if (state != State.Stopped && state != State.Destroyed) { - try { - stopRouter(router.getId(), false); - } catch (ResourceUnavailableException e) { - s_logger.warn("Fail to stop router " + router.getInstanceName(), e); - throw new ConnectionException(false, "Fail to stop router " + router.getInstanceName()); - } catch (ConcurrentOperationException e) { - s_logger.warn("Fail to stop router " + router.getInstanceName(), e); - throw new ConnectionException(false, "Fail to stop router " + router.getInstanceName()); - } - } - router.setStopPending(false); - router = _routerDao.persist(router); - } - } - } - - @Override - public AgentControlAnswer processControlCommand(long agentId, AgentControlCommand cmd) { - return null; - } - - @Override - - public boolean processDisconnect(long agentId, Status state) { - return false; - } - - @Override - public boolean processTimeout(long agentId, long seq) { - return false; - } - - protected String getRouterControlIp(long routerId) { - String routerControlIpAddress = null; - List nics = _nicDao.listByVmId(routerId); - for (NicVO n : nics) { - NetworkVO nc = _networkDao.findById(n.getNetworkId()); - if (nc.getTrafficType() == TrafficType.Control) { - routerControlIpAddress = n.getIp4Address(); - } - } - - if(routerControlIpAddress == null) { - s_logger.warn("Unable to find router's control ip in its attached NICs!. routerId: " + routerId); - DomainRouterVO router = _routerDao.findById(routerId); - return router.getPrivateIpAddress(); - } - - return routerControlIpAddress; - } - - - protected String getRouterIpInNetwork(long networkId, long instanceId) { - return _nicDao.getIpAddress(networkId, instanceId); - } - - - @Override - public boolean plugNic(Network network, NicTO nic, VirtualMachineTO vm, ReservationContext context, DeployDestination dest) - throws ConcurrentOperationException, ResourceUnavailableException, - InsufficientCapacityException { - //not supported - throw new UnsupportedOperationException("Plug nic is not supported for vm of type " + vm.getType()); - } - - @Override - public boolean unplugNic(Network network, NicTO nic, VirtualMachineTO vm, ReservationContext context, DeployDestination dest) - throws ConcurrentOperationException, ResourceUnavailableException { - //not supported - throw new UnsupportedOperationException("Unplug nic is not supported for vm of type " + vm.getType()); - } - - @Override - public void prepareStop(VirtualMachineProfile profile){ - //Collect network usage before stopping Vm - VMInstanceVO vm = profile.getVirtualMachine(); + return _routerDao.findById(router.getId()); + } else { + return null; + } + } - DomainRouterVO router = _routerDao.findById(vm.getId()); - if(router == null){ - return; - } - - String privateIP = router.getPrivateIpAddress(); - - if (privateIP != null) { - List routerNics = _nicDao.listByVmId(router.getId()); - for (Nic routerNic : routerNics) { - Network network = _networkMgr.getNetwork(routerNic.getNetworkId()); - if (network.getTrafficType() == TrafficType.Public) { - boolean forVpc = router.getVpcId() != null; - final NetworkUsageCommand usageCmd = new NetworkUsageCommand(privateIP, router.getHostName(), - forVpc, routerNic.getIp4Address()); - UserStatisticsVO previousStats = _statsDao.findBy(router.getAccountId(), - router.getDataCenterIdToDeployIn(), network.getId(), null, router.getId(), router.getType().toString()); - NetworkUsageAnswer answer = null; - try { - answer = (NetworkUsageAnswer) _agentMgr.easySend(router.getHostId(), usageCmd); - } catch (Exception e) { - s_logger.warn("Error while collecting network stats from router: "+router.getInstanceName()+" from host: "+router.getHostId(), e); - continue; - } - - if (answer != null) { - if (!answer.getResult()) { - s_logger.warn("Error while collecting network stats from router: "+router.getInstanceName()+" from host: "+router.getHostId() + "; details: " + answer.getDetails()); - continue; - } - Transaction txn = Transaction.open(Transaction.CLOUD_DB); - try { - if ((answer.getBytesReceived() == 0) && (answer.getBytesSent() == 0)) { - s_logger.debug("Recieved and Sent bytes are both 0. Not updating user_statistics"); - continue; - } - txn.start(); - UserStatisticsVO stats = _statsDao.lock(router.getAccountId(), - router.getDataCenterIdToDeployIn(), network.getId(), null, router.getId(), router.getType().toString()); - if (stats == null) { - s_logger.warn("unable to find stats for account: " + router.getAccountId()); - continue; - } - - if(previousStats != null - && ((previousStats.getCurrentBytesReceived() != stats.getCurrentBytesReceived()) - || (previousStats.getCurrentBytesSent() != stats.getCurrentBytesSent()))){ - s_logger.debug("Router stats changed from the time NetworkUsageCommand was sent. " + - "Ignoring current answer. Router: "+answer.getRouterName()+" Rcvd: " + - answer.getBytesReceived()+ "Sent: " +answer.getBytesSent()); - continue; - } - - if (stats.getCurrentBytesReceived() > answer.getBytesReceived()) { - if (s_logger.isDebugEnabled()) { - s_logger.debug("Received # of bytes that's less than the last one. " + - "Assuming something went wrong and persisting it. Router: " + - answer.getRouterName()+" Reported: " + answer.getBytesReceived() - + " Stored: " + stats.getCurrentBytesReceived()); - } - stats.setNetBytesReceived(stats.getNetBytesReceived() + stats.getCurrentBytesReceived()); - } - stats.setCurrentBytesReceived(answer.getBytesReceived()); - if (stats.getCurrentBytesSent() > answer.getBytesSent()) { - if (s_logger.isDebugEnabled()) { - s_logger.debug("Received # of bytes that's less than the last one. " + - "Assuming something went wrong and persisting it. Router: " + - answer.getRouterName()+" Reported: " + answer.getBytesSent() - + " Stored: " + stats.getCurrentBytesSent()); - } - stats.setNetBytesSent(stats.getNetBytesSent() + stats.getCurrentBytesSent()); - } - stats.setCurrentBytesSent(answer.getBytesSent()); - _statsDao.update(stats.getId(), stats); - txn.commit(); - } catch (Exception e) { - txn.rollback(); - s_logger.warn("Unable to update user statistics for account: " + router.getAccountId() - + " Rx: " + answer.getBytesReceived() + "; Tx: " + answer.getBytesSent()); - } finally { - txn.close(); - } - } - } - } - } - } -} + @Override + public DomainRouterVO stop(VirtualRouter router, boolean forced, User user, Account caller) throws ConcurrentOperationException, ResourceUnavailableException { + s_logger.debug("Stopping router " + router); + try { + if (_itMgr.advanceStop((DomainRouterVO) router, forced, user, caller)) { + return _routerDao.findById(router.getId()); + } else { + return null; + } + } catch (OperationTimedoutException e) { + throw new CloudRuntimeException("Unable to stop " + router, e); + } + } + + @Override + public boolean applyDhcpEntry(Network network, final NicProfile nic, VirtualMachineProfile profile, + DeployDestination dest, List routers) + throws ResourceUnavailableException { + _userVmDao.loadDetails((UserVmVO) profile.getVirtualMachine()); + + final VirtualMachineProfile updatedProfile = profile; + final boolean isZoneBasic = (dest.getDataCenter().getNetworkType() == NetworkType.Basic); + final Long podId = isZoneBasic ? dest.getPod().getId() : null; + + boolean podLevelException = false; + //for user vm in Basic zone we should try to re-deploy vm in a diff pod if it fails to deploy in original pod; so throwing exception with Pod scope + if (isZoneBasic && podId != null && updatedProfile.getVirtualMachine().getType() == VirtualMachine.Type.User + && network.getTrafficType() == TrafficType.Guest && network.getGuestType() == Network.GuestType.Shared) { + podLevelException = true; + } + + return applyRules(network, routers, "dhcp entry", podLevelException, podId, true, new RuleApplier() { + @Override + public boolean execute(Network network, VirtualRouter router) throws ResourceUnavailableException { + //for basic zone, send dhcp/dns information to all routers in the basic network only when _dnsBasicZoneUpdates is set to "all" value + Commands cmds = new Commands(OnError.Stop); + if (!(isZoneBasic && router.getPodIdToDeployIn().longValue() != podId.longValue() && _dnsBasicZoneUpdates.equalsIgnoreCase("pod"))) { + NicVO nicVo = _nicDao.findById(nic.getId()); + createDhcpEntryCommand(router, updatedProfile.getVirtualMachine(), nicVo, cmds); + return sendCommandsToRouter(router, cmds); + } + return true; + } + }); + } + + private String findDefaultDnsIp(long userVmId) { + NicVO defaultNic = _nicDao.findDefaultNicForVM(userVmId); + + //check if DNS provider is the domR + if (!_networkMgr.isProviderSupportServiceInNetwork(defaultNic.getNetworkId(), Service.Dns, Provider.VirtualRouter)) { + return null; + } + + NetworkOfferingVO offering = _networkOfferingDao.findById(_networkDao.findById(defaultNic.getNetworkId()).getNetworkOfferingId()); + if (offering.getRedundantRouter()) { + return findGatewayIp(userVmId); + } + + //find domR's nic in the network + NicVO domrDefaultNic = _nicDao.findByNetworkIdAndType(defaultNic.getNetworkId(), VirtualMachine.Type.DomainRouter); + return domrDefaultNic.getIp4Address(); + } + + private String findGatewayIp(long userVmId) { + NicVO defaultNic = _nicDao.findDefaultNicForVM(userVmId); + return defaultNic.getGateway(); + } + + @Override + public boolean applyUserData(Network network, final NicProfile nic, VirtualMachineProfile profile, DeployDestination dest, List routers) + throws ResourceUnavailableException { + _userVmDao.loadDetails((UserVmVO) profile.getVirtualMachine()); + + final VirtualMachineProfile updatedProfile = profile; + final boolean isZoneBasic = (dest.getDataCenter().getNetworkType() == NetworkType.Basic); + final Long podId = isZoneBasic ? dest.getPod().getId() : null; + + boolean podLevelException = false; + //for user vm in Basic zone we should try to re-deploy vm in a diff pod if it fails to deploy in original pod; so throwing exception with Pod scope + if (isZoneBasic && podId != null && updatedProfile.getVirtualMachine().getType() == VirtualMachine.Type.User + && network.getTrafficType() == TrafficType.Guest && network.getGuestType() == Network.GuestType.Shared) { + podLevelException = true; + } + + return applyRules(network, routers, "userdata and password entry", podLevelException, podId, false, new RuleApplier() { + @Override + public boolean execute(Network network, VirtualRouter router) throws ResourceUnavailableException { + //for basic zone, send vm data/password information only to the router in the same pod + Commands cmds = new Commands(OnError.Stop); + if (!(isZoneBasic && router.getPodIdToDeployIn().longValue() != podId.longValue())) { + NicVO nicVo = _nicDao.findById(nic.getId()); + createPasswordCommand(router, updatedProfile, nicVo, cmds); + createVmDataCommand(router, updatedProfile.getVirtualMachine(), nicVo, updatedProfile.getVirtualMachine().getDetail("SSH.PublicKey"), cmds); + return sendCommandsToRouter(router, cmds); + } + return true; + } + }); + } + + @Override + public DomainRouterVO persist(DomainRouterVO router) { + DomainRouterVO virtualRouter = _routerDao.persist(router); + return virtualRouter; + } + + @Override + //FIXME add partial success and STOP state support + public String[] applyVpnUsers(Network network, List users, List routers) throws ResourceUnavailableException { + if (routers == null || routers.isEmpty()) { + s_logger.warn("Failed to add/remove VPN users: no router found for account and zone"); + throw new ResourceUnavailableException("Unable to assign ip addresses, domR doesn't exist for network " + + network.getId(), DataCenter.class, network.getDataCenterId()); + } + + boolean agentResults = true; + + for (DomainRouterVO router : routers) { + if (router.getState() != State.Running) { + s_logger.warn("Failed to add/remove VPN users: router not in running state"); + throw new ResourceUnavailableException("Unable to assign ip addresses, domR is not in right state " + + router.getState(), DataCenter.class, network.getDataCenterId()); + } + + Commands cmds = new Commands(OnError.Continue); + List addUsers = new ArrayList(); + List removeUsers = new ArrayList(); + for (VpnUser user : users) { + if (user.getState() == VpnUser.State.Add || user.getState() == VpnUser.State.Active) { + addUsers.add(user); + } else if (user.getState() == VpnUser.State.Revoke) { + removeUsers.add(user); + } + } + + VpnUsersCfgCommand cmd = new VpnUsersCfgCommand(addUsers, removeUsers); + cmd.setAccessDetail(NetworkElementCommand.ACCOUNT_ID, String.valueOf(router.getAccountId())); + cmd.setAccessDetail(NetworkElementCommand.GUEST_NETWORK_CIDR, network.getCidr()); + cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); + cmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(network.getId(), router.getId())); + cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); + DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); + cmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); + + cmds.addCommand(cmd); + + + // Currently we receive just one answer from the agent. In the future we have to parse individual answers and set + // results accordingly + boolean agentResult = sendCommandsToRouter(router, cmds); + agentResults = agentResults && agentResult; + } + + String[] result = new String[users.size()]; + for (int i = 0; i < result.length; i++) { + if (agentResults) { + result[i] = null; + } else { + result[i] = String.valueOf(agentResults); + } + } + + return result; + } + + @Override + public DomainRouterVO findById(long id) { + return _routerDao.findById(id); + } + + @Override + public DomainRouterVO findByName(String name) { + if (!VirtualMachineName.isValidRouterName(name)) { + return null; + } + + return _routerDao.findById(VirtualMachineName.getRouterId(name)); + } + + @Override @ActionEvent(eventType = EventTypes.EVENT_ROUTER_START, eventDescription = "starting router Vm", async = true) + public VirtualRouter startRouter(long id) throws ResourceUnavailableException, InsufficientCapacityException, ConcurrentOperationException{ + return startRouter(id, true); + } + + @Override + public VirtualRouter startRouter(long routerId, boolean reprogramNetwork) throws ResourceUnavailableException, + InsufficientCapacityException, ConcurrentOperationException { + Account caller = UserContext.current().getCaller(); + User callerUser = _accountMgr.getActiveUser(UserContext.current().getCallerUserId()); + + // verify parameters + DomainRouterVO router = _routerDao.findById(routerId); + if (router == null) { + throw new InvalidParameterValueException("Unable to find router by id " + routerId + "."); + } + _accountMgr.checkAccess(caller, null, true, router); + + Account owner = _accountMgr.getAccount(router.getAccountId()); + + // Check if all networks are implemented for the domR; if not - implement them + DataCenter dc = _dcDao.findById(router.getDataCenterIdToDeployIn()); + HostPodVO pod = null; + if (router.getPodIdToDeployIn() != null) { + pod = _podDao.findById(router.getPodIdToDeployIn()); + } + DeployDestination dest = new DeployDestination(dc, pod, null, null); + + ReservationContext context = new ReservationContextImpl(null, null, callerUser, owner); + + List nics = _nicDao.listByVmId(routerId); + + for (NicVO nic : nics) { + if (!_networkMgr.startNetwork(nic.getNetworkId(), dest, context)) { + s_logger.warn("Failed to start network id=" + nic.getNetworkId() + " as a part of domR start"); + throw new CloudRuntimeException("Failed to start network id=" + nic.getNetworkId() + " as a part of domR start"); + } + } + + UserVO user = _userDao.findById(UserContext.current().getCallerUserId()); + Map params = new HashMap(); + if (reprogramNetwork) { + params.put(Param.ReProgramGuestNetworks, true); + } else { + params.put(Param.ReProgramGuestNetworks, false); + } + VirtualRouter virtualRouter = startVirtualRouter(router, user, caller, params); + if(virtualRouter == null){ + throw new CloudRuntimeException("Failed to start router with id " + routerId); + } + return virtualRouter; + } + + private void createAssociateIPCommands(final VirtualRouter router, final List ips, Commands cmds, long vmId) { + + // Ensure that in multiple vlans case we first send all ip addresses of vlan1, then all ip addresses of vlan2, etc.. + Map> vlanIpMap = new HashMap>(); + for (final PublicIpAddress ipAddress : ips) { + String vlanTag = ipAddress.getVlanTag(); + ArrayList ipList = vlanIpMap.get(vlanTag); + if (ipList == null) { + ipList = new ArrayList(); + } + //domR doesn't support release for sourceNat IP address; so reset the state + if (ipAddress.isSourceNat() && ipAddress.getState() == IpAddress.State.Releasing) { + ipAddress.setState(IpAddress.State.Allocated); + } + ipList.add(ipAddress); + vlanIpMap.put(vlanTag, ipList); + } + + for (Map.Entry> vlanAndIp : vlanIpMap.entrySet()) { + List ipAddrList = vlanAndIp.getValue(); + // Source nat ip address should always be sent first + Collections.sort(ipAddrList, new Comparator() { + @Override + public int compare(PublicIpAddress o1, PublicIpAddress o2) { + boolean s1 = o1.isSourceNat(); + boolean s2 = o2.isSourceNat(); + return (s1 ^ s2) ? ((s1 ^ true) ? 1 : -1) : 0; + } + }); + + // Get network rate - required for IpAssoc + Integer networkRate = _networkMgr.getNetworkRate(ipAddrList.get(0).getNetworkId(), router.getId()); + Network network = _networkMgr.getNetwork(ipAddrList.get(0).getNetworkId()); + + IpAddressTO[] ipsToSend = new IpAddressTO[ipAddrList.size()]; + int i = 0; + boolean firstIP = true; + + for (final PublicIpAddress ipAddr : ipAddrList) { + + boolean add = (ipAddr.getState() == IpAddress.State.Releasing ? false : true); + boolean sourceNat = ipAddr.isSourceNat(); + /* enable sourceNAT for the first ip of the public interface */ + if (firstIP) { + sourceNat = true; + } + String vlanId = ipAddr.getVlanTag(); + String vlanGateway = ipAddr.getGateway(); + String vlanNetmask = ipAddr.getNetmask(); + String vifMacAddress = ipAddr.getMacAddress(); + + String vmGuestAddress = null; + + IpAddressTO ip = new IpAddressTO(ipAddr.getAccountId(), ipAddr.getAddress().addr(), add, firstIP, + sourceNat, vlanId, vlanGateway, vlanNetmask, vifMacAddress, vmGuestAddress, networkRate, ipAddr.isOneToOneNat()); + + ip.setTrafficType(network.getTrafficType()); + ip.setNetworkName(_networkMgr.getNetworkTag(router.getHypervisorType(), network)); + ipsToSend[i++] = ip; + /* send the firstIP = true for the first Add, this is to create primary on interface*/ + if (!firstIP || add) { + firstIP = false; + } + } + IpAssocCommand cmd = new IpAssocCommand(ipsToSend); + cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); + cmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(ipAddrList.get(0).getNetworkId(), router.getId())); + cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); + DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); + cmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); + + cmds.addCommand("IPAssocCommand", cmd); + } + } + + private void createApplyPortForwardingRulesCommands(List rules, VirtualRouter router, Commands cmds, long guestNetworkId) { + List rulesTO = null; + if (rules != null) { + rulesTO = new ArrayList(); + for (PortForwardingRule rule : rules) { + IpAddress sourceIp = _networkMgr.getIp(rule.getSourceIpAddressId()); + PortForwardingRuleTO ruleTO = new PortForwardingRuleTO(rule, null, sourceIp.getAddress().addr()); + rulesTO.add(ruleTO); + } + } + + SetPortForwardingRulesCommand cmd = null; + + if (router.getVpcId() != null) { + cmd = new SetPortForwardingRulesVpcCommand(rulesTO); + } else { + cmd = new SetPortForwardingRulesCommand(rulesTO); + } + + cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); + cmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(guestNetworkId, router.getId())); + cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); + DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); + cmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); + + cmds.addCommand(cmd); + } + + private void createApplyStaticNatRulesCommands(List rules, VirtualRouter router, Commands cmds, long guestNetworkId) { + List rulesTO = null; + if (rules != null) { + rulesTO = new ArrayList(); + for (StaticNatRule rule : rules) { + IpAddress sourceIp = _networkMgr.getIp(rule.getSourceIpAddressId()); + StaticNatRuleTO ruleTO = new StaticNatRuleTO(rule, null, sourceIp.getAddress().addr(), rule.getDestIpAddress()); + rulesTO.add(ruleTO); + } + } + + SetStaticNatRulesCommand cmd = new SetStaticNatRulesCommand(rulesTO, router.getVpcId()); + cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); + cmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(guestNetworkId, router.getId())); + cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); + DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); + cmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); + cmds.addCommand(cmd); + } + + private void createApplyLoadBalancingRulesCommands(List rules, VirtualRouter router, Commands cmds, long guestNetworkId) { + + LoadBalancerTO[] lbs = new LoadBalancerTO[rules.size()]; + int i = 0; + for (LoadBalancingRule rule : rules) { + boolean revoked = (rule.getState().equals(FirewallRule.State.Revoke)); + String protocol = rule.getProtocol(); + String algorithm = rule.getAlgorithm(); + + String srcIp = _networkMgr.getIp(rule.getSourceIpAddressId()).getAddress().addr(); + int srcPort = rule.getSourcePortStart(); + List destinations = rule.getDestinations(); + List stickinessPolicies = rule.getStickinessPolicies(); + LoadBalancerTO lb = new LoadBalancerTO(srcIp, srcPort, protocol, algorithm, revoked, false, destinations, stickinessPolicies); + lbs[i++] = lb; + } + String routerPublicIp = null; + + if (router instanceof DomainRouterVO) { + DomainRouterVO domr = _routerDao.findById(router.getId()); + routerPublicIp = domr.getPublicIpAddress(); + } + + Network guestNetwork = _networkMgr.getNetwork(guestNetworkId); + Nic nic = _nicDao.findByInstanceIdAndNetworkId(guestNetwork.getId(), router.getId()); + NicProfile nicProfile = new NicProfile(nic, guestNetwork, nic.getBroadcastUri(), nic.getIsolationUri(), + _networkMgr.getNetworkRate(guestNetwork.getId(), router.getId()), + _networkMgr.isSecurityGroupSupportedInNetwork(guestNetwork), + _networkMgr.getNetworkTag(router.getHypervisorType(), guestNetwork)); + + LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(lbs,routerPublicIp, + getRouterIpInNetwork(guestNetworkId, router.getId()),router.getPrivateIpAddress(), + _itMgr.toNicTO(nicProfile, router.getHypervisorType()), router.getVpcId()); + + cmd.lbStatsVisibility = _configDao.getValue(Config.NetworkLBHaproxyStatsVisbility.key()); + cmd.lbStatsUri = _configDao.getValue(Config.NetworkLBHaproxyStatsUri.key()); + cmd.lbStatsAuth = _configDao.getValue(Config.NetworkLBHaproxyStatsAuth.key()); + cmd.lbStatsPort = _configDao.getValue(Config.NetworkLBHaproxyStatsPort.key()); + + + cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); + cmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(guestNetworkId, router.getId())); + cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); + DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); + cmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); + cmds.addCommand(cmd); + + } + + private void createApplyVpnCommands(RemoteAccessVpn vpn, VirtualRouter router, Commands cmds) { + List vpnUsers = _vpnUsersDao.listByAccount(vpn.getAccountId()); + List addUsers = new ArrayList(); + List removeUsers = new ArrayList(); + for (VpnUser user : vpnUsers) { + if (user.getState() == VpnUser.State.Add) { + addUsers.add(user); + } else if (user.getState() == VpnUser.State.Revoke) { + removeUsers.add(user); + } + } + + VpnUsersCfgCommand addUsersCmd = new VpnUsersCfgCommand(addUsers, removeUsers); + addUsersCmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); + addUsersCmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(vpn.getNetworkId(), router.getId())); + addUsersCmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); + + IpAddress ip = _networkMgr.getIp(vpn.getServerAddressId()); + + RemoteAccessVpnCfgCommand startVpnCmd = new RemoteAccessVpnCfgCommand(true, ip.getAddress().addr(), + vpn.getLocalIp(), vpn.getIpRange(), vpn.getIpsecPresharedKey()); + startVpnCmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); + startVpnCmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(vpn.getNetworkId(), router.getId())); + startVpnCmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); + DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); + startVpnCmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); + + cmds.addCommand("users", addUsersCmd); + cmds.addCommand("startVpn", startVpnCmd); + } + + private void createPasswordCommand(VirtualRouter router, VirtualMachineProfile profile, NicVO nic, Commands cmds) { + String password = (String) profile.getParameter(VirtualMachineProfile.Param.VmPassword); + DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); + + // password should be set only on default network element + if (password != null && nic.isDefaultNic()) { + final String encodedPassword = PasswordGenerator.rot13(password); + SavePasswordCommand cmd = new SavePasswordCommand(encodedPassword, nic.getIp4Address(), profile.getVirtualMachine().getHostName()); + cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); + cmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(nic.getNetworkId(), router.getId())); + cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); + cmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); + + cmds.addCommand("password", cmd); + } + + } + + private void createVmDataCommand(VirtualRouter router, UserVm vm, NicVO nic, String publicKey, Commands cmds) { + String serviceOffering = _serviceOfferingDao.findByIdIncludingRemoved(vm.getServiceOfferingId()).getDisplayText(); + String zoneName = _dcDao.findById(router.getDataCenterIdToDeployIn()).getName(); + cmds.addCommand("vmdata", + generateVmDataCommand(router, nic.getIp4Address(), vm.getUserData(), serviceOffering, zoneName, nic.getIp4Address(), + vm.getHostName(), vm.getInstanceName(), vm.getId(), publicKey, nic.getNetworkId())); + + } + + private void createVmDataCommandForVMs(DomainRouterVO router, Commands cmds, long guestNetworkId) { + List vms = _userVmDao.listByNetworkIdAndStates(guestNetworkId, State.Running, State.Migrating, State.Stopping); + DataCenterVO dc = _dcDao.findById(router.getDataCenterIdToDeployIn()); + for (UserVmVO vm : vms) { + boolean createVmData = true; + if (dc.getNetworkType() == NetworkType.Basic && router.getPodIdToDeployIn().longValue() != vm.getPodIdToDeployIn().longValue()) { + createVmData = false; + } + + if (createVmData) { + NicVO nic = _nicDao.findByInstanceIdAndNetworkId(guestNetworkId, vm.getId()); + if (nic != null) { + s_logger.debug("Creating user data entry for vm " + vm + " on domR " + router); + createVmDataCommand(router, vm, nic, null, cmds); + } + } + } + } + + private void createDhcpEntryCommand(VirtualRouter router, UserVm vm, NicVO nic, Commands cmds) { + DhcpEntryCommand dhcpCommand = new DhcpEntryCommand(nic.getMacAddress(), nic.getIp4Address(), vm.getHostName()); + DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); + String gatewayIp = findGatewayIp(vm.getId()); + if (!gatewayIp.equals(nic.getGateway())) { + GuestOSVO guestOS = _guestOSDao.findById(vm.getGuestOSId()); + // Don't set dhcp:router option for non-default nic on CentOS/RHEL, because they would set routing on wrong interface + // This is tricky, we may need to update this when we have more information on various OS's behavior + if (guestOS.getDisplayName().startsWith("CentOS") || guestOS.getDisplayName().startsWith("Red Hat Enterprise")) { + gatewayIp = "0.0.0.0"; + } + } + dhcpCommand.setDefaultRouter(gatewayIp); + dhcpCommand.setDefaultDns(findDefaultDnsIp(vm.getId())); + + dhcpCommand.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); + dhcpCommand.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); + dhcpCommand.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(nic.getNetworkId(), router.getId())); + dhcpCommand.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); + + cmds.addCommand("dhcp", dhcpCommand); + } + + private void createDhcpEntryCommandsForVMs(DomainRouterVO router, Commands cmds, long guestNetworkId) { + List vms = _userVmDao.listByNetworkIdAndStates(guestNetworkId, State.Running, State.Migrating, State.Stopping); + DataCenterVO dc = _dcDao.findById(router.getDataCenterIdToDeployIn()); + for (UserVmVO vm : vms) { + boolean createDhcp = true; + if (dc.getNetworkType() == NetworkType.Basic && router.getPodIdToDeployIn().longValue() != vm.getPodIdToDeployIn().longValue() + && _dnsBasicZoneUpdates.equalsIgnoreCase("pod")) { + createDhcp = false; + } + if (createDhcp) { + NicVO nic = _nicDao.findByInstanceIdAndNetworkId(guestNetworkId, vm.getId()); + if (nic != null) { + s_logger.debug("Creating dhcp entry for vm " + vm + " on domR " + router + "."); + createDhcpEntryCommand(router, vm, nic, cmds); + } + } + } + } + + protected boolean sendCommandsToRouter(final VirtualRouter router, Commands cmds) throws AgentUnavailableException { + Answer[] answers = null; + try { + answers = _agentMgr.send(router.getHostId(), cmds); + } catch (OperationTimedoutException e) { + s_logger.warn("Timed Out", e); + throw new AgentUnavailableException("Unable to send commands to virtual router ", router.getHostId(), e); + } + + if (answers == null) { + return false; + } + + if (answers.length != cmds.size()) { + return false; + } + + // FIXME: Have to return state for individual command in the future + boolean result = true; + if (answers.length > 0) { + for (Answer answer : answers) { + if (!answer.getResult()) { + result = false; + break; + } + } + } + return result; + } + + protected void handleSingleWorkingRedundantRouter(List connectedRouters, List disconnectedRouters, String reason) throws ResourceUnavailableException + { + if (connectedRouters.isEmpty() || disconnectedRouters.isEmpty()) { + return; + } + if (connectedRouters.size() != 1 || disconnectedRouters.size() != 1) { + s_logger.warn("How many redundant routers do we have?? "); + return; + } + if (!connectedRouters.get(0).getIsRedundantRouter()) { + throw new ResourceUnavailableException("Who is calling this with non-redundant router or non-domain router?", + DataCenter.class, connectedRouters.get(0).getDataCenterIdToDeployIn()); + } + if (!disconnectedRouters.get(0).getIsRedundantRouter()) { + throw new ResourceUnavailableException("Who is calling this with non-redundant router or non-domain router?", + DataCenter.class, disconnectedRouters.get(0).getDataCenterIdToDeployIn()); + } + + DomainRouterVO connectedRouter = (DomainRouterVO)connectedRouters.get(0); + DomainRouterVO disconnectedRouter = (DomainRouterVO)disconnectedRouters.get(0); + + if (s_logger.isDebugEnabled()) { + s_logger.debug("About to stop the router " + disconnectedRouter.getInstanceName() + " due to: " + reason); + } + String title = "Virtual router " + disconnectedRouter.getInstanceName() + " would be stopped after connecting back, due to " + reason; + String context = "Virtual router (name: " + disconnectedRouter.getInstanceName() + ", id: " + disconnectedRouter.getId() + ") would be stopped after connecting back, due to: " + reason; + _alertMgr.sendAlert(AlertManager.ALERT_TYPE_DOMAIN_ROUTER, + disconnectedRouter.getDataCenterIdToDeployIn(), disconnectedRouter.getPodIdToDeployIn(), title, context); + disconnectedRouter.setStopPending(true); + disconnectedRouter = _routerDao.persist(disconnectedRouter); + + int connRouterPR = getRealPriority(connectedRouter); + int disconnRouterPR = getRealPriority(disconnectedRouter); + if (connRouterPR < disconnRouterPR) { + //connRouterPR < disconnRouterPR, they won't equal at anytime + if (!connectedRouter.getIsPriorityBumpUp()) { + final BumpUpPriorityCommand command = new BumpUpPriorityCommand(); + command.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(connectedRouter.getId())); + command.setAccessDetail(NetworkElementCommand.ROUTER_NAME, connectedRouter.getInstanceName()); + final Answer answer = _agentMgr.easySend(connectedRouter.getHostId(), command); + if (!answer.getResult()) { + s_logger.error("Failed to bump up " + connectedRouter.getInstanceName() + "'s priority! " + answer.getDetails()); + } + } else { + String t = "Can't bump up virtual router " + connectedRouter.getInstanceName() + "'s priority due to it's already bumped up!"; + _alertMgr.sendAlert(AlertManager.ALERT_TYPE_DOMAIN_ROUTER, + connectedRouter.getDataCenterIdToDeployIn(), connectedRouter.getPodIdToDeployIn(), t, t); + } + } + } + + @Override + public boolean associatePublicIP(Network network, final List ipAddress, List routers) + throws ResourceUnavailableException { + if (ipAddress == null || ipAddress.isEmpty()) { + s_logger.debug("No ip association rules to be applied for network " + network.getId()); + return true; + } + return applyRules(network, routers, "ip association", false, null, false, new RuleApplier() { + @Override + public boolean execute(Network network, VirtualRouter router) throws ResourceUnavailableException { + Commands cmds = new Commands(OnError.Continue); + createAssociateIPCommands(router, ipAddress, cmds, 0); + return sendCommandsToRouter(router, cmds); + } + }); + } + + @Override + public boolean applyFirewallRules(Network network, final List rules, List routers) throws ResourceUnavailableException { + if (rules == null || rules.isEmpty()) { + s_logger.debug("No firewall rules to be applied for network " + network.getId()); + return true; + } + return applyRules(network, routers, "firewall rules", false, null, false, new RuleApplier() { + @Override + public boolean execute(Network network, VirtualRouter router) throws ResourceUnavailableException { + if (rules.get(0).getPurpose() == Purpose.LoadBalancing) { + // for load balancer we have to resend all lb rules for the network + List lbs = _loadBalancerDao.listByNetworkId(network.getId()); + List lbRules = new ArrayList(); + for (LoadBalancerVO lb : lbs) { + List dstList = _lbMgr.getExistingDestinations(lb.getId()); + List policyList = _lbMgr.getStickinessPolicies(lb.getId()); + LoadBalancingRule loadBalancing = new LoadBalancingRule(lb, dstList,policyList); + lbRules.add(loadBalancing); + } + return sendLBRules(router, lbRules, network.getId()); + } else if (rules.get(0).getPurpose() == Purpose.PortForwarding) { + return sendPortForwardingRules(router, (List) rules, network.getId()); + } else if (rules.get(0).getPurpose() == Purpose.StaticNat) { + return sendStaticNatRules(router, (List) rules, network.getId()); + } else if (rules.get(0).getPurpose() == Purpose.Firewall) { + return sendFirewallRules(router, (List) rules, network.getId()); + } else { + s_logger.warn("Unable to apply rules of purpose: " + rules.get(0).getPurpose()); + return false; + } + } + }); + } + + protected boolean sendLBRules(VirtualRouter router, List rules, long guestNetworkId) throws ResourceUnavailableException { + Commands cmds = new Commands(OnError.Continue); + createApplyLoadBalancingRulesCommands(rules, router, cmds, guestNetworkId); + return sendCommandsToRouter(router, cmds); + } + + protected boolean sendPortForwardingRules(VirtualRouter router, List rules, long guestNetworkId) throws ResourceUnavailableException { + Commands cmds = new Commands(OnError.Continue); + createApplyPortForwardingRulesCommands(rules, router, cmds, guestNetworkId); + return sendCommandsToRouter(router, cmds); + } + + protected boolean sendStaticNatRules(VirtualRouter router, List rules, long guestNetworkId) throws ResourceUnavailableException { + Commands cmds = new Commands(OnError.Continue); + createApplyStaticNatRulesCommands(rules, router, cmds, guestNetworkId); + return sendCommandsToRouter(router, cmds); + } + + @Override + public List getRoutersForNetwork(long networkId) { + List routers = _routerDao.findByNetwork(networkId); + List vrs = new ArrayList(routers.size()); + for (DomainRouterVO router : routers) { + vrs.add(router); + } + return vrs; + } + + private void createFirewallRulesCommands(List rules, VirtualRouter router, Commands cmds, long guestNetworkId) { + List rulesTO = null; + if (rules != null) { + rulesTO = new ArrayList(); + for (FirewallRule rule : rules) { + IpAddress sourceIp = _networkMgr.getIp(rule.getSourceIpAddressId()); + FirewallRuleTO ruleTO = new FirewallRuleTO(rule, null, sourceIp.getAddress().addr()); + rulesTO.add(ruleTO); + } + } + + SetFirewallRulesCommand cmd = new SetFirewallRulesCommand(rulesTO); + cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); + cmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(guestNetworkId, router.getId())); + cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); + DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); + cmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); + cmds.addCommand(cmd); + } + + + protected boolean sendFirewallRules(VirtualRouter router, List rules, long guestNetworkId) throws ResourceUnavailableException { + Commands cmds = new Commands(OnError.Continue); + createFirewallRulesCommands(rules, router, cmds, guestNetworkId); + return sendCommandsToRouter(router, cmds); + } + + @Override + public String getDnsBasicZoneUpdate() { + return _dnsBasicZoneUpdates; + } + + protected interface RuleApplier { + boolean execute(Network network, VirtualRouter router) throws ResourceUnavailableException; + } + + protected boolean applyRules(Network network, List routers, String typeString, + boolean isPodLevelException, Long podId, boolean failWhenDisconnect, RuleApplier applier) throws ResourceUnavailableException { + if (routers == null || routers.isEmpty()) { + s_logger.warn("Unable to apply " + typeString + ", virtual router doesn't exist in the network " + network.getId()); + throw new ResourceUnavailableException("Unable to apply " + typeString , DataCenter.class, network.getDataCenterId()); + } + + DataCenter dc = _dcDao.findById(network.getDataCenterId()); + boolean isZoneBasic = (dc.getNetworkType() == NetworkType.Basic); + + // isPodLevelException and podId is only used for basic zone + assert !((!isZoneBasic && isPodLevelException) || (isZoneBasic && isPodLevelException && podId == null)); + + List connectedRouters = new ArrayList(); + List disconnectedRouters = new ArrayList(); + boolean result = true; + String msg = "Unable to apply " + typeString + " on disconnected router "; + for (VirtualRouter router : routers) { + if (router.getState() == State.Running) { + s_logger.debug("Applying " + typeString + " in network " + network); + + if (router.isStopPending()) { + if (_hostDao.findById(router.getHostId()).getStatus() == Status.Up) { + throw new ResourceUnavailableException("Unable to process due to the stop pending router " + + router.getInstanceName() + " haven't been stopped after it's host coming back!", + DataCenter.class, router.getDataCenterIdToDeployIn()); + } + s_logger.debug("Router " + router.getInstanceName() + " is stop pending, so not sending apply " + + typeString + " commands to the backend"); + continue; + } + try { + result = applier.execute(network, router); + connectedRouters.add(router); + } catch (AgentUnavailableException e) { + s_logger.warn(msg + router.getInstanceName(), e); + disconnectedRouters.add(router); + } + + //If rules fail to apply on one domR and not due to disconnection, no need to proceed with the rest + if (!result) { + if (isZoneBasic && isPodLevelException) { + throw new ResourceUnavailableException("Unable to apply " + typeString + " on router ", Pod.class, podId); + } + throw new ResourceUnavailableException("Unable to apply " + typeString + " on router ", DataCenter.class, + router.getDataCenterIdToDeployIn()); + } + + } else if (router.getState() == State.Stopped || router.getState() == State.Stopping) { + s_logger.debug("Router " + router.getInstanceName() + " is in " + router.getState() + + ", so not sending apply " + typeString + " commands to the backend"); + } else { + s_logger.warn("Unable to apply " + typeString +", virtual router is not in the right state " + router.getState()); + if (isZoneBasic && isPodLevelException) { + throw new ResourceUnavailableException("Unable to apply " + typeString + + ", virtual router is not in the right state", Pod.class, podId); + } + throw new ResourceUnavailableException("Unable to apply " + typeString + + ", virtual router is not in the right state", DataCenter.class, router.getDataCenterIdToDeployIn()); + } + } + + if (!connectedRouters.isEmpty()) { + if (!isZoneBasic && !disconnectedRouters.isEmpty() && disconnectedRouters.get(0).getIsRedundantRouter()) { + // These disconnected redundant virtual routers are out of sync now, stop them for synchronization + handleSingleWorkingRedundantRouter(connectedRouters, disconnectedRouters, msg); + } + } else if (!disconnectedRouters.isEmpty()) { + for (VirtualRouter router : disconnectedRouters) { + if (s_logger.isDebugEnabled()) { + s_logger.debug(msg + router.getInstanceName() + "(" + router.getId() + ")"); + } + } + if (isZoneBasic && isPodLevelException) { + throw new ResourceUnavailableException(msg, Pod.class, podId); + } + throw new ResourceUnavailableException(msg, DataCenter.class, disconnectedRouters.get(0).getDataCenterIdToDeployIn()); + } + + result = true; + if (failWhenDisconnect) { + result = !connectedRouters.isEmpty(); + } + return result; + } + + @Override + public boolean applyStaticNats(Network network, final List rules, List routers) throws ResourceUnavailableException { + if (rules == null || rules.isEmpty()) { + s_logger.debug("No static nat rules to be applied for network " + network.getId()); + return true; + } + return applyRules(network, routers, "static nat rules", false, null, false, new RuleApplier() { + @Override + public boolean execute(Network network, VirtualRouter router) throws ResourceUnavailableException { + return applyStaticNat(router, rules, network.getId()); + } + }); + } + + + protected boolean applyStaticNat(VirtualRouter router, List rules, long guestNetworkId) throws ResourceUnavailableException { + Commands cmds = new Commands(OnError.Continue); + createApplyStaticNatCommands(rules, router, cmds, guestNetworkId); + return sendCommandsToRouter(router, cmds); + } + + private void createApplyStaticNatCommands(List rules, VirtualRouter router, Commands cmds, + long guestNetworkId) { + List rulesTO = null; + if (rules != null) { + rulesTO = new ArrayList(); + for (StaticNat rule : rules) { + IpAddress sourceIp = _networkMgr.getIp(rule.getSourceIpAddressId()); + StaticNatRuleTO ruleTO = new StaticNatRuleTO(0, sourceIp.getAddress().addr(), null, + null, rule.getDestIpAddress(), null, null, null, rule.isForRevoke(), false); + rulesTO.add(ruleTO); + } + } + + SetStaticNatRulesCommand cmd = new SetStaticNatRulesCommand(rulesTO, router.getVpcId()); + cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); + cmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(guestNetworkId, router.getId())); + cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); + DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); + cmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); + cmds.addCommand(cmd); + } + + @Override + public int getTimeout() { + return -1; + } + + @Override + public boolean isRecurring() { + return false; + } + + @Override + public boolean processAnswers(long agentId, long seq, Answer[] answers) { + return false; + } + + @Override + public boolean processCommands(long agentId, long seq, Command[] commands) { + return false; + } + + @Override + public void processConnect(HostVO host, StartupCommand cmd, boolean forRebalance) throws ConnectionException { + UserContext context = UserContext.current(); + context.setAccountId(1); + List routers = _routerDao.listIsolatedByHostId(host.getId()); + for (DomainRouterVO router : routers) { + if (router.isStopPending()) { + State state = router.getState(); + if (state != State.Stopped && state != State.Destroyed) { + try { + stopRouter(router.getId(), false); + } catch (ResourceUnavailableException e) { + s_logger.warn("Fail to stop router " + router.getInstanceName(), e); + throw new ConnectionException(false, "Fail to stop router " + router.getInstanceName()); + } catch (ConcurrentOperationException e) { + s_logger.warn("Fail to stop router " + router.getInstanceName(), e); + throw new ConnectionException(false, "Fail to stop router " + router.getInstanceName()); + } + } + router.setStopPending(false); + router = _routerDao.persist(router); + } + } + } + + @Override + public AgentControlAnswer processControlCommand(long agentId, AgentControlCommand cmd) { + return null; + } + + @Override + + public boolean processDisconnect(long agentId, Status state) { + return false; + } + + @Override + public boolean processTimeout(long agentId, long seq) { + return false; + } + + protected String getRouterControlIp(long routerId) { + String routerControlIpAddress = null; + List nics = _nicDao.listByVmId(routerId); + for (NicVO n : nics) { + NetworkVO nc = _networkDao.findById(n.getNetworkId()); + if (nc.getTrafficType() == TrafficType.Control) { + routerControlIpAddress = n.getIp4Address(); + } + } + + if(routerControlIpAddress == null) { + s_logger.warn("Unable to find router's control ip in its attached NICs!. routerId: " + routerId); + DomainRouterVO router = _routerDao.findById(routerId); + return router.getPrivateIpAddress(); + } + + return routerControlIpAddress; + } + + + protected String getRouterIpInNetwork(long networkId, long instanceId) { + return _nicDao.getIpAddress(networkId, instanceId); + } + + + @Override + public boolean plugNic(Network network, NicTO nic, VirtualMachineTO vm, ReservationContext context, DeployDestination dest) + throws ConcurrentOperationException, ResourceUnavailableException, + InsufficientCapacityException { + //not supported + throw new UnsupportedOperationException("Plug nic is not supported for vm of type " + vm.getType()); + } + + @Override + public boolean unplugNic(Network network, NicTO nic, VirtualMachineTO vm, ReservationContext context, DeployDestination dest) + throws ConcurrentOperationException, ResourceUnavailableException { + //not supported + throw new UnsupportedOperationException("Unplug nic is not supported for vm of type " + vm.getType()); + } + + @Override + public void prepareStop(VirtualMachineProfile profile){ + //Collect network usage before stopping Vm + VMInstanceVO vm = profile.getVirtualMachine(); + + DomainRouterVO router = _routerDao.findById(vm.getId()); + if(router == null){ + return; + } + + String privateIP = router.getPrivateIpAddress(); + + if (privateIP != null) { + List routerNics = _nicDao.listByVmId(router.getId()); + for (Nic routerNic : routerNics) { + Network network = _networkMgr.getNetwork(routerNic.getNetworkId()); + if (network.getTrafficType() == TrafficType.Public) { + boolean forVpc = router.getVpcId() != null; + final NetworkUsageCommand usageCmd = new NetworkUsageCommand(privateIP, router.getHostName(), + forVpc, routerNic.getIp4Address()); + UserStatisticsVO previousStats = _statsDao.findBy(router.getAccountId(), + router.getDataCenterIdToDeployIn(), network.getId(), null, router.getId(), router.getType().toString()); + NetworkUsageAnswer answer = null; + try { + answer = (NetworkUsageAnswer) _agentMgr.easySend(router.getHostId(), usageCmd); + } catch (Exception e) { + s_logger.warn("Error while collecting network stats from router: "+router.getInstanceName()+" from host: "+router.getHostId(), e); + continue; + } + + if (answer != null) { + if (!answer.getResult()) { + s_logger.warn("Error while collecting network stats from router: "+router.getInstanceName()+" from host: "+router.getHostId() + "; details: " + answer.getDetails()); + continue; + } + Transaction txn = Transaction.open(Transaction.CLOUD_DB); + try { + if ((answer.getBytesReceived() == 0) && (answer.getBytesSent() == 0)) { + s_logger.debug("Recieved and Sent bytes are both 0. Not updating user_statistics"); + continue; + } + txn.start(); + UserStatisticsVO stats = _statsDao.lock(router.getAccountId(), + router.getDataCenterIdToDeployIn(), network.getId(), null, router.getId(), router.getType().toString()); + if (stats == null) { + s_logger.warn("unable to find stats for account: " + router.getAccountId()); + continue; + } + + if(previousStats != null + && ((previousStats.getCurrentBytesReceived() != stats.getCurrentBytesReceived()) + || (previousStats.getCurrentBytesSent() != stats.getCurrentBytesSent()))){ + s_logger.debug("Router stats changed from the time NetworkUsageCommand was sent. " + + "Ignoring current answer. Router: "+answer.getRouterName()+" Rcvd: " + + answer.getBytesReceived()+ "Sent: " +answer.getBytesSent()); + continue; + } + + if (stats.getCurrentBytesReceived() > answer.getBytesReceived()) { + if (s_logger.isDebugEnabled()) { + s_logger.debug("Received # of bytes that's less than the last one. " + + "Assuming something went wrong and persisting it. Router: " + + answer.getRouterName()+" Reported: " + answer.getBytesReceived() + + " Stored: " + stats.getCurrentBytesReceived()); + } + stats.setNetBytesReceived(stats.getNetBytesReceived() + stats.getCurrentBytesReceived()); + } + stats.setCurrentBytesReceived(answer.getBytesReceived()); + if (stats.getCurrentBytesSent() > answer.getBytesSent()) { + if (s_logger.isDebugEnabled()) { + s_logger.debug("Received # of bytes that's less than the last one. " + + "Assuming something went wrong and persisting it. Router: " + + answer.getRouterName()+" Reported: " + answer.getBytesSent() + + " Stored: " + stats.getCurrentBytesSent()); + } + stats.setNetBytesSent(stats.getNetBytesSent() + stats.getCurrentBytesSent()); + } + stats.setCurrentBytesSent(answer.getBytesSent()); + _statsDao.update(stats.getId(), stats); + txn.commit(); + } catch (Exception e) { + txn.rollback(); + s_logger.warn("Unable to update user statistics for account: " + router.getAccountId() + + " Rx: " + answer.getBytesReceived() + "; Tx: " + answer.getBytesSent()); + } finally { + txn.close(); + } + } + } + } + } + } +}