// Licensed to the Apache Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, // software distributed under the License is distributed on an // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. package com.cloud.network.router; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; import java.util.Calendar; import java.util.Collections; import java.util.Comparator; import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; import java.util.TimeZone; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import javax.ejb.Local; import javax.inject.Inject; import javax.naming.ConfigurationException; import org.apache.log4j.Logger; import org.apache.cloudstack.alert.AlertService; import org.apache.cloudstack.alert.AlertService.AlertType; import org.apache.cloudstack.api.command.admin.router.RebootRouterCmd; import org.apache.cloudstack.api.command.admin.router.UpgradeRouterCmd; import org.apache.cloudstack.api.command.admin.router.UpgradeRouterTemplateCmd; import org.apache.cloudstack.config.ApiServiceConfiguration; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; import org.apache.cloudstack.framework.config.ConfigDepot; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.framework.jobs.AsyncJobManager; import org.apache.cloudstack.framework.jobs.impl.AsyncJobVO; import org.apache.cloudstack.managed.context.ManagedContextRunnable; import org.apache.cloudstack.utils.identity.ManagementServerNode; import com.cloud.agent.AgentManager; import com.cloud.agent.Listener; import com.cloud.agent.api.AgentControlAnswer; import com.cloud.agent.api.AgentControlCommand; import com.cloud.agent.api.Answer; import com.cloud.agent.api.BumpUpPriorityCommand; import com.cloud.agent.api.CheckRouterAnswer; import com.cloud.agent.api.CheckRouterCommand; import com.cloud.agent.api.CheckS2SVpnConnectionsAnswer; import com.cloud.agent.api.CheckS2SVpnConnectionsCommand; import com.cloud.agent.api.Command; import com.cloud.agent.api.GetDomRVersionAnswer; import com.cloud.agent.api.GetDomRVersionCmd; import com.cloud.agent.api.GetRouterAlertsAnswer; import com.cloud.agent.api.ModifySshKeysCommand; import com.cloud.agent.api.NetworkUsageAnswer; import com.cloud.agent.api.NetworkUsageCommand; import com.cloud.agent.api.PvlanSetupCommand; import com.cloud.agent.api.StartupCommand; import com.cloud.agent.api.check.CheckSshCommand; import com.cloud.agent.api.routing.AggregationControlCommand; import com.cloud.agent.api.routing.AggregationControlCommand.Action; import com.cloud.agent.api.routing.CreateIpAliasCommand; import com.cloud.agent.api.routing.DeleteIpAliasCommand; import com.cloud.agent.api.routing.DhcpEntryCommand; import com.cloud.agent.api.routing.DnsMasqConfigCommand; import com.cloud.agent.api.routing.GetRouterAlertsCommand; import com.cloud.agent.api.routing.IpAliasTO; import com.cloud.agent.api.routing.IpAssocCommand; import com.cloud.agent.api.routing.LoadBalancerConfigCommand; import com.cloud.agent.api.routing.NetworkElementCommand; import com.cloud.agent.api.routing.RemoteAccessVpnCfgCommand; import com.cloud.agent.api.routing.SavePasswordCommand; import com.cloud.agent.api.routing.SetFirewallRulesCommand; import com.cloud.agent.api.routing.SetMonitorServiceCommand; import com.cloud.agent.api.routing.SetPortForwardingRulesCommand; import com.cloud.agent.api.routing.SetPortForwardingRulesVpcCommand; import com.cloud.agent.api.routing.SetStaticNatRulesCommand; import com.cloud.agent.api.routing.VmDataCommand; import com.cloud.agent.api.routing.VpnUsersCfgCommand; import com.cloud.agent.api.to.DhcpTO; import com.cloud.agent.api.to.FirewallRuleTO; import com.cloud.agent.api.to.IpAddressTO; import com.cloud.agent.api.to.LoadBalancerTO; import com.cloud.agent.api.to.MonitorServiceTO; import com.cloud.agent.api.to.PortForwardingRuleTO; import com.cloud.agent.api.to.StaticNatRuleTO; import com.cloud.agent.manager.Commands; import com.cloud.alert.AlertManager; import com.cloud.api.ApiAsyncJobDispatcher; import com.cloud.api.ApiGsonHelper; import com.cloud.cluster.ManagementServerHostVO; import com.cloud.cluster.dao.ManagementServerHostDao; import com.cloud.configuration.Config; import com.cloud.configuration.ConfigurationManager; import com.cloud.configuration.ZoneConfig; import com.cloud.dc.ClusterVO; import com.cloud.dc.DataCenter; import com.cloud.dc.DataCenter.NetworkType; import com.cloud.dc.DataCenterVO; import com.cloud.dc.HostPodVO; import com.cloud.dc.Pod; import com.cloud.dc.Vlan; import com.cloud.dc.VlanVO; import com.cloud.dc.dao.ClusterDao; import com.cloud.dc.dao.DataCenterDao; import com.cloud.dc.dao.HostPodDao; import com.cloud.dc.dao.VlanDao; import com.cloud.deploy.DataCenterDeployment; import com.cloud.deploy.DeployDestination; import com.cloud.deploy.DeploymentPlan; import com.cloud.deploy.DeploymentPlanner.ExcludeList; import com.cloud.event.ActionEvent; import com.cloud.event.EventTypes; import com.cloud.exception.AgentUnavailableException; import com.cloud.exception.ConcurrentOperationException; import com.cloud.exception.ConnectionException; import com.cloud.exception.InsufficientAddressCapacityException; import com.cloud.exception.InsufficientCapacityException; import com.cloud.exception.InsufficientServerCapacityException; import com.cloud.exception.InsufficientVirtualNetworkCapcityException; import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.OperationTimedoutException; import com.cloud.exception.ResourceUnavailableException; import com.cloud.exception.StorageUnavailableException; import com.cloud.host.Host; import com.cloud.host.HostVO; import com.cloud.host.Status; import com.cloud.host.dao.HostDao; import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.maint.Version; import com.cloud.network.IpAddress; import com.cloud.network.IpAddressManager; import com.cloud.network.MonitoringService; import com.cloud.network.Network; import com.cloud.network.Network.GuestType; import com.cloud.network.Network.Provider; import com.cloud.network.Network.Service; import com.cloud.network.NetworkModel; import com.cloud.network.NetworkService; import com.cloud.network.Networks.BroadcastDomainType; import com.cloud.network.Networks.IsolationType; import com.cloud.network.Networks.TrafficType; import com.cloud.network.PhysicalNetworkServiceProvider; import com.cloud.network.PublicIpAddress; import com.cloud.network.RemoteAccessVpn; import com.cloud.network.Site2SiteCustomerGateway; import com.cloud.network.Site2SiteVpnConnection; import com.cloud.network.SshKeysDistriMonitor; import com.cloud.network.VirtualNetworkApplianceService; import com.cloud.network.VirtualRouterProvider; import com.cloud.network.VirtualRouterProvider.Type; import com.cloud.network.VpnUser; import com.cloud.network.VpnUserVO; import com.cloud.network.addr.PublicIp; import com.cloud.network.dao.FirewallRulesDao; import com.cloud.network.dao.IPAddressDao; import com.cloud.network.dao.IPAddressVO; import com.cloud.network.dao.LoadBalancerDao; import com.cloud.network.dao.LoadBalancerVMMapDao; import com.cloud.network.dao.LoadBalancerVO; import com.cloud.network.dao.MonitoringServiceDao; import com.cloud.network.dao.MonitoringServiceVO; import com.cloud.network.dao.NetworkDao; import com.cloud.network.dao.NetworkVO; import com.cloud.network.dao.OpRouterMonitorServiceDao; import com.cloud.network.dao.OpRouterMonitorServiceVO; import com.cloud.network.dao.PhysicalNetworkServiceProviderDao; import com.cloud.network.dao.RemoteAccessVpnDao; import com.cloud.network.dao.Site2SiteCustomerGatewayDao; import com.cloud.network.dao.Site2SiteVpnConnectionDao; import com.cloud.network.dao.Site2SiteVpnConnectionVO; import com.cloud.network.dao.Site2SiteVpnGatewayDao; import com.cloud.network.dao.UserIpv6AddressDao; import com.cloud.network.dao.VirtualRouterProviderDao; import com.cloud.network.dao.VpnUserDao; import com.cloud.network.lb.LoadBalancingRule; import com.cloud.network.lb.LoadBalancingRule.LbDestination; import com.cloud.network.lb.LoadBalancingRule.LbHealthCheckPolicy; import com.cloud.network.lb.LoadBalancingRule.LbSslCert; import com.cloud.network.lb.LoadBalancingRule.LbStickinessPolicy; import com.cloud.network.lb.LoadBalancingRulesManager; import com.cloud.network.router.VirtualRouter.RedundantState; import com.cloud.network.router.VirtualRouter.Role; import com.cloud.network.rules.FirewallRule; import com.cloud.network.rules.FirewallRule.Purpose; import com.cloud.network.rules.FirewallRuleVO; import com.cloud.network.rules.LoadBalancerContainer.Scheme; import com.cloud.network.rules.PortForwardingRule; import com.cloud.network.rules.RulesManager; import com.cloud.network.rules.StaticNat; import com.cloud.network.rules.StaticNatImpl; import com.cloud.network.rules.StaticNatRule; import com.cloud.network.rules.dao.PortForwardingRulesDao; import com.cloud.network.vpn.Site2SiteVpnManager; import com.cloud.offering.NetworkOffering; import com.cloud.offering.ServiceOffering; import com.cloud.offerings.NetworkOfferingVO; import com.cloud.offerings.dao.NetworkOfferingDao; import com.cloud.resource.ResourceManager; import com.cloud.server.ConfigurationServer; import com.cloud.service.ServiceOfferingVO; import com.cloud.service.dao.ServiceOfferingDao; import com.cloud.storage.Storage.ProvisioningType; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.Volume; import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.GuestOSDao; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VolumeDao; import com.cloud.user.Account; import com.cloud.user.AccountManager; import com.cloud.user.User; import com.cloud.user.UserStatisticsVO; import com.cloud.user.UserStatsLogVO; import com.cloud.user.UserVO; import com.cloud.user.dao.UserDao; import com.cloud.user.dao.UserStatisticsDao; import com.cloud.user.dao.UserStatsLogDao; import com.cloud.uservm.UserVm; import com.cloud.utils.NumbersUtil; import com.cloud.utils.Pair; import com.cloud.utils.PasswordGenerator; import com.cloud.utils.StringUtils; import com.cloud.utils.component.ComponentContext; import com.cloud.utils.component.ManagerBase; import com.cloud.utils.concurrency.NamedThreadFactory; import com.cloud.utils.db.DB; import com.cloud.utils.db.EntityManager; import com.cloud.utils.db.Filter; import com.cloud.utils.db.GlobalLock; import com.cloud.utils.db.JoinBuilder; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.Transaction; import com.cloud.utils.db.TransactionCallbackNoReturn; import com.cloud.utils.db.TransactionStatus; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.fsm.StateListener; import com.cloud.utils.net.Ip; import com.cloud.utils.net.MacAddress; import com.cloud.utils.net.NetUtils; import com.cloud.vm.DomainRouterVO; import com.cloud.vm.Nic; import com.cloud.vm.NicIpAlias; import com.cloud.vm.NicProfile; import com.cloud.vm.NicVO; import com.cloud.vm.ReservationContext; import com.cloud.vm.ReservationContextImpl; import com.cloud.vm.UserVmVO; import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachine.State; import com.cloud.vm.VirtualMachineGuru; import com.cloud.vm.VirtualMachineManager; import com.cloud.vm.VirtualMachineName; import com.cloud.vm.VirtualMachineProfile; import com.cloud.vm.VirtualMachineProfile.Param; import com.cloud.vm.dao.DomainRouterDao; import com.cloud.vm.dao.NicDao; import com.cloud.vm.dao.NicIpAliasDao; import com.cloud.vm.dao.NicIpAliasVO; import com.cloud.vm.dao.UserVmDao; import com.cloud.vm.dao.UserVmDetailsDao; import com.cloud.vm.dao.VMInstanceDao; /** * VirtualNetworkApplianceManagerImpl manages the different types of virtual network appliances available in the Cloud Stack. */ @Local(value = { VirtualNetworkApplianceManager.class, VirtualNetworkApplianceService.class }) public class VirtualNetworkApplianceManagerImpl extends ManagerBase implements VirtualNetworkApplianceManager, VirtualNetworkApplianceService, VirtualMachineGuru, Listener, Configurable, StateListener { private static final Logger s_logger = Logger.getLogger(VirtualNetworkApplianceManagerImpl.class); @Inject EntityManager _entityMgr; @Inject DataCenterDao _dcDao = null; @Inject VlanDao _vlanDao = null; @Inject FirewallRulesDao _rulesDao = null; @Inject LoadBalancerDao _loadBalancerDao = null; @Inject LoadBalancerVMMapDao _loadBalancerVMMapDao = null; @Inject IPAddressDao _ipAddressDao = null; @Inject VMTemplateDao _templateDao = null; @Inject DomainRouterDao _routerDao = null; @Inject UserDao _userDao = null; @Inject UserStatisticsDao _userStatsDao = null; @Inject HostDao _hostDao = null; @Inject ConfigurationDao _configDao; @Inject HostPodDao _podDao = null; @Inject UserStatsLogDao _userStatsLogDao = null; @Inject AgentManager _agentMgr; @Inject AlertManager _alertMgr; @Inject AccountManager _accountMgr; @Inject ConfigurationManager _configMgr; @Inject ConfigurationServer _configServer; @Inject ServiceOfferingDao _serviceOfferingDao = null; @Inject UserVmDao _userVmDao; @Inject VMInstanceDao _vmDao; @Inject NetworkOfferingDao _networkOfferingDao = null; @Inject GuestOSDao _guestOSDao = null; @Inject NetworkOrchestrationService _networkMgr; @Inject NetworkModel _networkModel; @Inject VirtualMachineManager _itMgr; @Inject VpnUserDao _vpnUsersDao; @Inject RulesManager _rulesMgr; @Inject NetworkDao _networkDao; @Inject LoadBalancingRulesManager _lbMgr; @Inject PortForwardingRulesDao _pfRulesDao; @Inject RemoteAccessVpnDao _vpnDao; @Inject NicDao _nicDao; @Inject NicIpAliasDao _nicIpAliasDao; @Inject VolumeDao _volumeDao = null; @Inject UserVmDetailsDao _vmDetailsDao; @Inject ClusterDao _clusterDao; @Inject ResourceManager _resourceMgr; @Inject PhysicalNetworkServiceProviderDao _physicalProviderDao; @Inject VirtualRouterProviderDao _vrProviderDao; @Inject ManagementServerHostDao _msHostDao; @Inject Site2SiteCustomerGatewayDao _s2sCustomerGatewayDao; @Inject Site2SiteVpnGatewayDao _s2sVpnGatewayDao; @Inject Site2SiteVpnConnectionDao _s2sVpnConnectionDao; @Inject Site2SiteVpnManager _s2sVpnMgr; @Inject UserIpv6AddressDao _ipv6Dao; @Inject NetworkService _networkSvc; @Inject IpAddressManager _ipAddrMgr; @Inject ConfigDepot _configDepot; @Inject MonitoringServiceDao _monitorServiceDao; @Inject AsyncJobManager _asyncMgr; @Inject protected ApiAsyncJobDispatcher _asyncDispatcher; @Inject OpRouterMonitorServiceDao _opRouterMonitorServiceDao; int _routerRamSize; int _routerCpuMHz; int _retry = 2; String _instance; String _mgmtCidr; int _routerStatsInterval = 300; int _routerCheckInterval = 30; int _rvrStatusUpdatePoolSize = 10; protected ServiceOfferingVO _offering; private String _dnsBasicZoneUpdates = "all"; private final Set _guestOSNeedGatewayOnNonDefaultNetwork = new HashSet(); private boolean _disableRpFilter = false; int _routerExtraPublicNics = 2; private int _usageAggregationRange = 1440; private String _usageTimeZone = "GMT"; private final long mgmtSrvrId = MacAddress.getMacAddress().toLong(); private static final int ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_COOPERATION = 5; // 5 seconds private static final int USAGE_AGGREGATION_RANGE_MIN = 10; // 10 minutes, same as com.cloud.usage.UsageManagerImpl.USAGE_AGGREGATION_RANGE_MIN private boolean _dailyOrHourly = false; ScheduledExecutorService _executor; ScheduledExecutorService _checkExecutor; ScheduledExecutorService _networkStatsUpdateExecutor; ExecutorService _rvrStatusUpdateExecutor; Account _systemAcct; BlockingQueue _vrUpdateQueue = null; @Override public boolean sendSshKeysToHost(final Long hostId, final String pubKey, final String prvKey) { final ModifySshKeysCommand cmd = new ModifySshKeysCommand(pubKey, prvKey); final Answer answer = _agentMgr.easySend(hostId, cmd); if (answer != null) { return true; } else { return false; } } @Override public VirtualRouter destroyRouter(final long routerId, final Account caller, final Long callerUserId) throws ResourceUnavailableException, ConcurrentOperationException { if (s_logger.isDebugEnabled()) { s_logger.debug("Attempting to destroy router " + routerId); } final DomainRouterVO router = _routerDao.findById(routerId); if (router == null) { return null; } _accountMgr.checkAccess(caller, null, true, router); _itMgr.expunge(router.getUuid()); _routerDao.remove(router.getId()); return router; } @Override @DB public VirtualRouter upgradeRouter(final UpgradeRouterCmd cmd) { final Long routerId = cmd.getId(); final Long serviceOfferingId = cmd.getServiceOfferingId(); final Account caller = CallContext.current().getCallingAccount(); final DomainRouterVO router = _routerDao.findById(routerId); if (router == null) { throw new InvalidParameterValueException("Unable to find router with id " + routerId); } _accountMgr.checkAccess(caller, null, true, router); if (router.getServiceOfferingId() == serviceOfferingId) { s_logger.debug("Router: " + routerId + "already has service offering: " + serviceOfferingId); return _routerDao.findById(routerId); } final ServiceOffering newServiceOffering = _entityMgr.findById(ServiceOffering.class, serviceOfferingId); if (newServiceOffering == null) { throw new InvalidParameterValueException("Unable to find service offering with id " + serviceOfferingId); } // check if it is a system service offering, if yes return with error as it cannot be used for user vms if (!newServiceOffering.getSystemUse()) { throw new InvalidParameterValueException("Cannot upgrade router vm to a non system service offering " + serviceOfferingId); } // Check that the router is stopped if (!router.getState().equals(State.Stopped)) { s_logger.warn("Unable to upgrade router " + router.toString() + " in state " + router.getState()); throw new InvalidParameterValueException("Unable to upgrade router " + router.toString() + " in state " + router.getState() + "; make sure the router is stopped and not in an error state before upgrading."); } final ServiceOfferingVO currentServiceOffering = _serviceOfferingDao.findById(router.getServiceOfferingId()); // Check that the service offering being upgraded to has the same storage pool preference as the VM's current service // offering if (currentServiceOffering.getUseLocalStorage() != newServiceOffering.getUseLocalStorage()) { throw new InvalidParameterValueException("Can't upgrade, due to new local storage status : " + newServiceOffering.getUseLocalStorage() + " is different from " + "curruent local storage status: " + currentServiceOffering.getUseLocalStorage()); } router.setServiceOfferingId(serviceOfferingId); if (_routerDao.update(routerId, router)) { return _routerDao.findById(routerId); } else { throw new CloudRuntimeException("Unable to upgrade router " + routerId); } } @Override public boolean savePasswordToRouter(final Network network, final NicProfile nic, final VirtualMachineProfile profile, final List routers) throws ResourceUnavailableException { _userVmDao.loadDetails((UserVmVO)profile.getVirtualMachine()); final VirtualMachineProfile updatedProfile = profile; return applyRules(network, routers, "save password entry", false, null, false, new RuleApplier() { @Override public boolean execute(final Network network, final VirtualRouter router) throws ResourceUnavailableException { // for basic zone, send vm data/password information only to the router in the same pod final Commands cmds = new Commands(Command.OnError.Stop); final NicVO nicVo = _nicDao.findById(nic.getId()); createPasswordCommand(router, updatedProfile, nicVo, cmds); return sendCommandsToRouter(router, cmds); } }); } @Override public boolean saveSSHPublicKeyToRouter(final Network network, final NicProfile nic, final VirtualMachineProfile profile, final List routers, final String sshPublicKey) throws ResourceUnavailableException { final UserVmVO vm = _userVmDao.findById(profile.getVirtualMachine().getId()); _userVmDao.loadDetails(vm); final VirtualMachineProfile updatedProfile = profile; return applyRules(network, routers, "save SSHkey entry", false, null, false, new RuleApplier() { @Override public boolean execute(final Network network, final VirtualRouter router) throws ResourceUnavailableException { // for basic zone, send vm data/password information only to the router in the same pod final Commands cmds = new Commands(Command.OnError.Stop); final NicVO nicVo = _nicDao.findById(nic.getId()); final VMTemplateVO template = _templateDao.findByIdIncludingRemoved(updatedProfile.getTemplateId()); if (template != null && template.getEnablePassword()) { createPasswordCommand(router, updatedProfile, nicVo, cmds); } createVmDataCommand(router, vm, nicVo, sshPublicKey, cmds); return sendCommandsToRouter(router, cmds); } }); } @Override public boolean saveUserDataToRouter(final Network network, final NicProfile nic, final VirtualMachineProfile profile, final List routers) throws ResourceUnavailableException { final UserVmVO vm = _userVmDao.findById(profile.getVirtualMachine().getId()); _userVmDao.loadDetails(vm); return applyRules(network, routers, "save userdata entry", false, null, false, new RuleApplier() { @Override public boolean execute(final Network network, final VirtualRouter router) throws ResourceUnavailableException { // for basic zone, send vm data/password information only to the router in the same pod final Commands cmds = new Commands(Command.OnError.Stop); final NicVO nicVo = _nicDao.findById(nic.getId()); createVmDataCommand(router, vm, nicVo, null, cmds); return sendCommandsToRouter(router, cmds); } }); } @Override @ActionEvent(eventType = EventTypes.EVENT_ROUTER_STOP, eventDescription = "stopping router Vm", async = true) public VirtualRouter stopRouter(final long routerId, final boolean forced) throws ResourceUnavailableException, ConcurrentOperationException { final CallContext context = CallContext.current(); final Account account = context.getCallingAccount(); // verify parameters DomainRouterVO router = _routerDao.findById(routerId); if (router == null) { throw new InvalidParameterValueException("Unable to find router by id " + routerId + "."); } _accountMgr.checkAccess(account, null, true, router); final UserVO user = _userDao.findById(CallContext.current().getCallingUserId()); final VirtualRouter virtualRouter = stop(router, forced, user, account); if (virtualRouter == null) { throw new CloudRuntimeException("Failed to stop router with id " + routerId); } // Clear stop pending flag after stopped successfully if (router.isStopPending()) { s_logger.info("Clear the stop pending flag of router " + router.getHostName() + " after stop router successfully"); router.setStopPending(false); router = _routerDao.persist(router); virtualRouter.setStopPending(false); } return virtualRouter; } @DB public void processStopOrRebootAnswer(final DomainRouterVO router, final Answer answer) { Transaction.execute(new TransactionCallbackNoReturn() { @Override public void doInTransactionWithoutResult(final TransactionStatus status) { //FIXME!!! - UserStats command should grab bytesSent/Received for all guest interfaces of the VR final List routerGuestNtwkIds = _routerDao.getRouterNetworks(router.getId()); for (final Long guestNtwkId : routerGuestNtwkIds) { final UserStatisticsVO userStats = _userStatsDao.lock(router.getAccountId(), router.getDataCenterId(), guestNtwkId, null, router.getId(), router.getType().toString()); if (userStats != null) { final long currentBytesRcvd = userStats.getCurrentBytesReceived(); userStats.setCurrentBytesReceived(0); userStats.setNetBytesReceived(userStats.getNetBytesReceived() + currentBytesRcvd); final long currentBytesSent = userStats.getCurrentBytesSent(); userStats.setCurrentBytesSent(0); userStats.setNetBytesSent(userStats.getNetBytesSent() + currentBytesSent); _userStatsDao.update(userStats.getId(), userStats); s_logger.debug("Successfully updated user statistics as a part of domR " + router + " reboot/stop"); } else { s_logger.warn("User stats were not created for account " + router.getAccountId() + " and dc " + router.getDataCenterId()); } } } }); } @Override @ActionEvent(eventType = EventTypes.EVENT_ROUTER_REBOOT, eventDescription = "rebooting router Vm", async = true) public VirtualRouter rebootRouter(final long routerId, final boolean reprogramNetwork) throws ConcurrentOperationException, ResourceUnavailableException, InsufficientCapacityException { final Account caller = CallContext.current().getCallingAccount(); // verify parameters final DomainRouterVO router = _routerDao.findById(routerId); if (router == null) { throw new InvalidParameterValueException("Unable to find domain router with id " + routerId + "."); } _accountMgr.checkAccess(caller, null, true, router); // Can reboot domain router only in Running state if (router == null || router.getState() != State.Running) { s_logger.warn("Unable to reboot, virtual router is not in the right state " + router.getState()); throw new ResourceUnavailableException("Unable to reboot domR, it is not in right state " + router.getState(), DataCenter.class, router.getDataCenterId()); } final UserVO user = _userDao.findById(CallContext.current().getCallingUserId()); s_logger.debug("Stopping and starting router " + router + " as a part of router reboot"); if (stop(router, false, user, caller) != null) { return startRouter(routerId, reprogramNetwork); } else { throw new CloudRuntimeException("Failed to reboot router " + router); } } static final ConfigKey UseExternalDnsServers = new ConfigKey(Boolean.class, "use.external.dns", "Advanced", "false", "Bypass internal dns, use external dns1 and dns2", true, ConfigKey.Scope.Zone, null); static final ConfigKey routerVersionCheckEnabled = new ConfigKey("Advanced", Boolean.class, "router.version.check", "true", "If true, router minimum required version is checked before sending command", false); @Override public boolean configure(final String name, final Map params) throws ConfigurationException { _executor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("RouterMonitor")); _checkExecutor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("RouterStatusMonitor")); _networkStatsUpdateExecutor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("NetworkStatsUpdater")); VirtualMachine.State.getStateMachine().registerListener(this); final Map configs = _configDao.getConfiguration("AgentManager", params); _routerRamSize = NumbersUtil.parseInt(configs.get("router.ram.size"), DEFAULT_ROUTER_VM_RAMSIZE); _routerCpuMHz = NumbersUtil.parseInt(configs.get("router.cpu.mhz"), DEFAULT_ROUTER_CPU_MHZ); _routerExtraPublicNics = NumbersUtil.parseInt(_configDao.getValue(Config.RouterExtraPublicNics.key()), 2); final String guestOSString = configs.get("network.dhcp.nondefaultnetwork.setgateway.guestos"); if (guestOSString != null) { final String[] guestOSList = guestOSString.split(","); for (final String os : guestOSList) { _guestOSNeedGatewayOnNonDefaultNetwork.add(os); } } String value = configs.get("start.retry"); _retry = NumbersUtil.parseInt(value, 2); value = configs.get("router.stats.interval"); _routerStatsInterval = NumbersUtil.parseInt(value, 300); value = configs.get("router.check.interval"); _routerCheckInterval = NumbersUtil.parseInt(value, 30); value = configs.get("router.check.poolsize"); _rvrStatusUpdatePoolSize = NumbersUtil.parseInt(value, 10); /* * We assume that one thread can handle 20 requests in 1 minute in normal situation, so here we give the queue size up to 50 minutes. * It's mostly for buffer, since each time CheckRouterTask running, it would add all the redundant networks in the queue immediately */ _vrUpdateQueue = new LinkedBlockingQueue(_rvrStatusUpdatePoolSize * 1000); _rvrStatusUpdateExecutor = Executors.newFixedThreadPool(_rvrStatusUpdatePoolSize, new NamedThreadFactory("RedundantRouterStatusMonitor")); _instance = configs.get("instance.name"); if (_instance == null) { _instance = "DEFAULT"; } final String rpValue = configs.get("network.disable.rpfilter"); if (rpValue != null && rpValue.equalsIgnoreCase("true")) { _disableRpFilter = true; } _dnsBasicZoneUpdates = String.valueOf(_configDao.getValue(Config.DnsBasicZoneUpdates.key())); s_logger.info("Router configurations: " + "ramsize=" + _routerRamSize); _agentMgr.registerForHostEvents(new SshKeysDistriMonitor(_agentMgr, _hostDao, _configDao), true, false, false); final boolean useLocalStorage = Boolean.parseBoolean(configs.get(Config.SystemVMUseLocalStorage.key())); _offering = new ServiceOfferingVO("System Offering For Software Router", 1, _routerRamSize, _routerCpuMHz, null, null, true, null, ProvisioningType.THIN, useLocalStorage, true, null, true, VirtualMachine.Type.DomainRouter, true); _offering.setUniqueName(ServiceOffering.routerDefaultOffUniqueName); _offering = _serviceOfferingDao.persistSystemServiceOffering(_offering); // this can sometimes happen, if DB is manually or programmatically manipulated if (_offering == null) { final String msg = "Data integrity problem : System Offering For Software router VM has been removed?"; s_logger.error(msg); throw new ConfigurationException(msg); } _systemAcct = _accountMgr.getSystemAccount(); final String aggregationRange = configs.get("usage.stats.job.aggregation.range"); _usageAggregationRange = NumbersUtil.parseInt(aggregationRange, 1440); _usageTimeZone = configs.get("usage.aggregation.timezone"); if (_usageTimeZone == null) { _usageTimeZone = "GMT"; } _agentMgr.registerForHostEvents(this, true, false, false); s_logger.info("DomainRouterManager is configured."); return true; } @Override public boolean start() { if (_routerStatsInterval > 0) { _executor.scheduleAtFixedRate(new NetworkUsageTask(), _routerStatsInterval, _routerStatsInterval, TimeUnit.SECONDS); } else { s_logger.debug("router.stats.interval - " + _routerStatsInterval + " so not scheduling the router stats thread"); } //Schedule Network stats update task final TimeZone usageTimezone = TimeZone.getTimeZone(_usageTimeZone); final Calendar cal = Calendar.getInstance(usageTimezone); cal.setTime(new Date()); long endDate = 0; final int HOURLY_TIME = 60; final int DAILY_TIME = 60 * 24; if (_usageAggregationRange == DAILY_TIME) { cal.roll(Calendar.DAY_OF_YEAR, false); cal.set(Calendar.HOUR_OF_DAY, 0); cal.set(Calendar.MINUTE, 0); cal.set(Calendar.SECOND, 0); cal.set(Calendar.MILLISECOND, 0); cal.roll(Calendar.DAY_OF_YEAR, true); cal.add(Calendar.MILLISECOND, -1); endDate = cal.getTime().getTime(); _dailyOrHourly = true; } else if (_usageAggregationRange == HOURLY_TIME) { cal.roll(Calendar.HOUR_OF_DAY, false); cal.set(Calendar.MINUTE, 0); cal.set(Calendar.SECOND, 0); cal.set(Calendar.MILLISECOND, 0); cal.roll(Calendar.HOUR_OF_DAY, true); cal.add(Calendar.MILLISECOND, -1); endDate = cal.getTime().getTime(); _dailyOrHourly = true; } else { endDate = cal.getTime().getTime(); _dailyOrHourly = false; } if (_usageAggregationRange < USAGE_AGGREGATION_RANGE_MIN) { s_logger.warn("Usage stats job aggregation range is to small, using the minimum value of " + USAGE_AGGREGATION_RANGE_MIN); _usageAggregationRange = USAGE_AGGREGATION_RANGE_MIN; } _networkStatsUpdateExecutor.scheduleAtFixedRate(new NetworkStatsUpdateTask(), (endDate - System.currentTimeMillis()), (_usageAggregationRange * 60 * 1000), TimeUnit.MILLISECONDS); if (_routerCheckInterval > 0) { _checkExecutor.scheduleAtFixedRate(new CheckRouterTask(), _routerCheckInterval, _routerCheckInterval, TimeUnit.SECONDS); for (int i = 0; i < _rvrStatusUpdatePoolSize; i++) { _rvrStatusUpdateExecutor.execute(new RvRStatusUpdateTask()); } } else { s_logger.debug("router.check.interval - " + _routerCheckInterval + " so not scheduling the redundant router checking thread"); } int _routerAlertsCheckInterval = RouterAlertsCheckInterval.value(); if (_routerAlertsCheckInterval > 0) { _checkExecutor.scheduleAtFixedRate(new CheckRouterAlertsTask(), _routerAlertsCheckInterval, _routerAlertsCheckInterval, TimeUnit.SECONDS); } else { s_logger.debug("router.alerts.check.interval - " + _routerAlertsCheckInterval + " so not scheduling the router alerts checking thread"); } return true; } @Override public boolean stop() { return true; } protected VirtualNetworkApplianceManagerImpl() { } private VmDataCommand generateVmDataCommand(final VirtualRouter router, final String vmPrivateIpAddress, final String userData, final String serviceOffering, final String zoneName, final String guestIpAddress, final String vmName, final String vmInstanceName, final long vmId, final String vmUuid, final String publicKey, final long guestNetworkId) { final VmDataCommand cmd = new VmDataCommand(vmPrivateIpAddress, vmName, _networkModel.getExecuteInSeqNtwkElmtCmd()); cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); cmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(guestNetworkId, router.getId())); cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); final DataCenterVO dcVo = _dcDao.findById(router.getDataCenterId()); cmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); cmd.addVmData("userdata", "user-data", userData); cmd.addVmData("metadata", "service-offering", StringUtils.unicodeEscape(serviceOffering)); cmd.addVmData("metadata", "availability-zone", StringUtils.unicodeEscape(zoneName)); cmd.addVmData("metadata", "local-ipv4", guestIpAddress); cmd.addVmData("metadata", "local-hostname", StringUtils.unicodeEscape(vmName)); if (dcVo.getNetworkType() == NetworkType.Basic) { cmd.addVmData("metadata", "public-ipv4", guestIpAddress); cmd.addVmData("metadata", "public-hostname", StringUtils.unicodeEscape(vmName)); } else { if (router.getPublicIpAddress() == null) { cmd.addVmData("metadata", "public-ipv4", guestIpAddress); } else { cmd.addVmData("metadata", "public-ipv4", router.getPublicIpAddress()); } cmd.addVmData("metadata", "public-hostname", router.getPublicIpAddress()); } if (vmUuid == null) { setVmInstanceId(vmInstanceName, vmId, cmd); } else { setVmInstanceId(vmUuid, cmd); } cmd.addVmData("metadata", "public-keys", publicKey); String cloudIdentifier = _configDao.getValue("cloud.identifier"); if (cloudIdentifier == null) { cloudIdentifier = ""; } else { cloudIdentifier = "CloudStack-{" + cloudIdentifier + "}"; } cmd.addVmData("metadata", "cloud-identifier", cloudIdentifier); return cmd; } private void setVmInstanceId(final String vmUuid, final VmDataCommand cmd) { cmd.addVmData("metadata", "instance-id", vmUuid); cmd.addVmData("metadata", "vm-id", vmUuid); } private void setVmInstanceId(final String vmInstanceName, final long vmId, final VmDataCommand cmd) { cmd.addVmData("metadata", "instance-id", vmInstanceName); cmd.addVmData("metadata", "vm-id", String.valueOf(vmId)); } protected class NetworkUsageTask extends ManagedContextRunnable { public NetworkUsageTask() { } @Override protected void runInContext() { try { final List routers = _routerDao.listByStateAndNetworkType(State.Running, GuestType.Isolated, mgmtSrvrId); s_logger.debug("Found " + routers.size() + " running routers. "); for (final DomainRouterVO router : routers) { final String privateIP = router.getPrivateIpAddress(); if (privateIP != null) { final boolean forVpc = router.getVpcId() != null; final List routerNics = _nicDao.listByVmId(router.getId()); for (final Nic routerNic : routerNics) { final Network network = _networkModel.getNetwork(routerNic.getNetworkId()); //Send network usage command for public nic in VPC VR //Send network usage command for isolated guest nic of non VPC VR if ((forVpc && network.getTrafficType() == TrafficType.Public) || (!forVpc && network.getTrafficType() == TrafficType.Guest && network.getGuestType() == Network.GuestType.Isolated)) { final NetworkUsageCommand usageCmd = new NetworkUsageCommand(privateIP, router.getHostName(), forVpc, routerNic.getIp4Address()); final String routerType = router.getType().toString(); final UserStatisticsVO previousStats = _userStatsDao.findBy(router.getAccountId(), router.getDataCenterId(), network.getId(), (forVpc ? routerNic.getIp4Address() : null), router.getId(), routerType); NetworkUsageAnswer answer = null; try { answer = (NetworkUsageAnswer)_agentMgr.easySend(router.getHostId(), usageCmd); } catch (final Exception e) { s_logger.warn("Error while collecting network stats from router: " + router.getInstanceName() + " from host: " + router.getHostId(), e); continue; } if (answer != null) { if (!answer.getResult()) { s_logger.warn("Error while collecting network stats from router: " + router.getInstanceName() + " from host: " + router.getHostId() + "; details: " + answer.getDetails()); continue; } try { if ((answer.getBytesReceived() == 0) && (answer.getBytesSent() == 0)) { s_logger.debug("Recieved and Sent bytes are both 0. Not updating user_statistics"); continue; } final NetworkUsageAnswer answerFinal = answer; Transaction.execute(new TransactionCallbackNoReturn() { @Override public void doInTransactionWithoutResult(final TransactionStatus status) { final UserStatisticsVO stats = _userStatsDao.lock(router.getAccountId(), router.getDataCenterId(), network.getId(), (forVpc ? routerNic.getIp4Address() : null), router.getId(), routerType); if (stats == null) { s_logger.warn("unable to find stats for account: " + router.getAccountId()); return; } if (previousStats != null && ((previousStats.getCurrentBytesReceived() != stats.getCurrentBytesReceived()) || (previousStats.getCurrentBytesSent() != stats.getCurrentBytesSent()))) { s_logger.debug("Router stats changed from the time NetworkUsageCommand was sent. " + "Ignoring current answer. Router: " + answerFinal.getRouterName() + " Rcvd: " + answerFinal.getBytesReceived() + "Sent: " + answerFinal.getBytesSent()); return; } if (stats.getCurrentBytesReceived() > answerFinal.getBytesReceived()) { if (s_logger.isDebugEnabled()) { s_logger.debug("Received # of bytes that's less than the last one. " + "Assuming something went wrong and persisting it. Router: " + answerFinal.getRouterName() + " Reported: " + answerFinal.getBytesReceived() + " Stored: " + stats.getCurrentBytesReceived()); } stats.setNetBytesReceived(stats.getNetBytesReceived() + stats.getCurrentBytesReceived()); } stats.setCurrentBytesReceived(answerFinal.getBytesReceived()); if (stats.getCurrentBytesSent() > answerFinal.getBytesSent()) { if (s_logger.isDebugEnabled()) { s_logger.debug("Received # of bytes that's less than the last one. " + "Assuming something went wrong and persisting it. Router: " + answerFinal.getRouterName() + " Reported: " + answerFinal.getBytesSent() + " Stored: " + stats.getCurrentBytesSent()); } stats.setNetBytesSent(stats.getNetBytesSent() + stats.getCurrentBytesSent()); } stats.setCurrentBytesSent(answerFinal.getBytesSent()); if (!_dailyOrHourly) { //update agg bytes stats.setAggBytesSent(stats.getNetBytesSent() + stats.getCurrentBytesSent()); stats.setAggBytesReceived(stats.getNetBytesReceived() + stats.getCurrentBytesReceived()); } _userStatsDao.update(stats.getId(), stats); } }); } catch (final Exception e) { s_logger.warn("Unable to update user statistics for account: " + router.getAccountId() + " Rx: " + answer.getBytesReceived() + "; Tx: " + answer.getBytesSent()); } } } } } } } catch (final Exception e) { s_logger.warn("Error while collecting network stats", e); } } } protected class NetworkStatsUpdateTask extends ManagedContextRunnable { public NetworkStatsUpdateTask() { } @Override protected void runInContext() { final GlobalLock scanLock = GlobalLock.getInternLock("network.stats"); try { if (scanLock.lock(ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_COOPERATION)) { //Check for ownership //msHost in UP state with min id should run the job final ManagementServerHostVO msHost = _msHostDao.findOneInUpState(new Filter(ManagementServerHostVO.class, "id", true, 0L, 1L)); if (msHost == null || (msHost.getMsid() != mgmtSrvrId)) { s_logger.debug("Skipping aggregate network stats update"); scanLock.unlock(); return; } try { Transaction.execute(new TransactionCallbackNoReturn() { @Override public void doInTransactionWithoutResult(final TransactionStatus status) { //get all stats with delta > 0 final List updatedStats = _userStatsDao.listUpdatedStats(); final Date updatedTime = new Date(); for (final UserStatisticsVO stat : updatedStats) { //update agg bytes stat.setAggBytesReceived(stat.getCurrentBytesReceived() + stat.getNetBytesReceived()); stat.setAggBytesSent(stat.getCurrentBytesSent() + stat.getNetBytesSent()); _userStatsDao.update(stat.getId(), stat); //insert into op_user_stats_log final UserStatsLogVO statsLog = new UserStatsLogVO(stat.getId(), stat.getNetBytesReceived(), stat.getNetBytesSent(), stat.getCurrentBytesReceived(), stat.getCurrentBytesSent(), stat.getAggBytesReceived(), stat.getAggBytesSent(), updatedTime); _userStatsLogDao.persist(statsLog); } s_logger.debug("Successfully updated aggregate network stats"); } }); } catch (final Exception e) { s_logger.debug("Failed to update aggregate network stats", e); } finally { scanLock.unlock(); } } } catch (final Exception e) { s_logger.debug("Exception while trying to acquire network stats lock", e); } finally { scanLock.releaseRef(); } } } @DB protected void updateSite2SiteVpnConnectionState(final List routers) { for (final DomainRouterVO router : routers) { final List conns = _s2sVpnMgr.getConnectionsForRouter(router); if (conns == null || conns.isEmpty()) { continue; } if (router.getState() != State.Running) { for (final Site2SiteVpnConnectionVO conn : conns) { if (conn.getState() != Site2SiteVpnConnection.State.Error) { conn.setState(Site2SiteVpnConnection.State.Disconnected); _s2sVpnConnectionDao.persist(conn); } } continue; } final List ipList = new ArrayList(); for (final Site2SiteVpnConnectionVO conn : conns) { if (conn.getState() != Site2SiteVpnConnection.State.Connected && conn.getState() != Site2SiteVpnConnection.State.Disconnected) { continue; } final Site2SiteCustomerGateway gw = _s2sCustomerGatewayDao.findById(conn.getCustomerGatewayId()); ipList.add(gw.getGatewayIp()); } final String privateIP = router.getPrivateIpAddress(); final HostVO host = _hostDao.findById(router.getHostId()); if (host == null || host.getState() != Status.Up) { continue; } else if (host.getManagementServerId() != ManagementServerNode.getManagementServerId()) { /* Only cover hosts managed by this management server */ continue; } else if (privateIP != null) { final CheckS2SVpnConnectionsCommand command = new CheckS2SVpnConnectionsCommand(ipList); command.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); command.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); command.setWait(30); final Answer origAnswer = _agentMgr.easySend(router.getHostId(), command); CheckS2SVpnConnectionsAnswer answer = null; if (origAnswer instanceof CheckS2SVpnConnectionsAnswer) { answer = (CheckS2SVpnConnectionsAnswer)origAnswer; } else { s_logger.warn("Unable to update router " + router.getHostName() + "'s VPN connection status"); continue; } if (!answer.getResult()) { s_logger.warn("Unable to update router " + router.getHostName() + "'s VPN connection status"); continue; } for (final Site2SiteVpnConnectionVO conn : conns) { final Site2SiteVpnConnectionVO lock = _s2sVpnConnectionDao.acquireInLockTable(conn.getId()); if (lock == null) { throw new CloudRuntimeException("Unable to acquire lock on " + lock); } try { if (conn.getState() != Site2SiteVpnConnection.State.Connected && conn.getState() != Site2SiteVpnConnection.State.Disconnected) { continue; } final Site2SiteVpnConnection.State oldState = conn.getState(); final Site2SiteCustomerGateway gw = _s2sCustomerGatewayDao.findById(conn.getCustomerGatewayId()); if (answer.isConnected(gw.getGatewayIp())) { conn.setState(Site2SiteVpnConnection.State.Connected); } else { conn.setState(Site2SiteVpnConnection.State.Disconnected); } _s2sVpnConnectionDao.persist(conn); if (oldState != conn.getState()) { final String title = "Site-to-site Vpn Connection to " + gw.getName() + " just switch from " + oldState + " to " + conn.getState(); final String context = "Site-to-site Vpn Connection to " + gw.getName() + " on router " + router.getHostName() + "(id: " + router.getId() + ") " + " just switch from " + oldState + " to " + conn.getState(); s_logger.info(context); _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_DOMAIN_ROUTER, router.getDataCenterId(), router.getPodIdToDeployIn(), title, context); } } finally { _s2sVpnConnectionDao.releaseFromLockTable(lock.getId()); } } } } } protected void updateRoutersRedundantState(final List routers) { boolean updated = false; for (final DomainRouterVO router : routers) { updated = false; if (!router.getIsRedundantRouter()) { continue; } final RedundantState prevState = router.getRedundantState(); if (router.getState() != State.Running) { router.setRedundantState(RedundantState.UNKNOWN); router.setIsPriorityBumpUp(false); updated = true; } else { final String privateIP = router.getPrivateIpAddress(); final HostVO host = _hostDao.findById(router.getHostId()); if (host == null || host.getState() != Status.Up) { router.setRedundantState(RedundantState.UNKNOWN); updated = true; } else if (privateIP != null) { final CheckRouterCommand command = new CheckRouterCommand(); command.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); command.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); command.setWait(30); final Answer origAnswer = _agentMgr.easySend(router.getHostId(), command); CheckRouterAnswer answer = null; if (origAnswer instanceof CheckRouterAnswer) { answer = (CheckRouterAnswer)origAnswer; } else { s_logger.warn("Unable to update router " + router.getHostName() + "'s status"); } RedundantState state = RedundantState.UNKNOWN; boolean isBumped = router.getIsPriorityBumpUp(); if (answer != null && answer.getResult()) { state = answer.getState(); isBumped = answer.isBumped(); } router.setRedundantState(state); router.setIsPriorityBumpUp(isBumped); updated = true; } } if (updated) { _routerDao.update(router.getId(), router); } final RedundantState currState = router.getRedundantState(); if (prevState != currState) { final String title = "Redundant virtual router " + router.getInstanceName() + " just switch from " + prevState + " to " + currState; final String context = "Redundant virtual router (name: " + router.getHostName() + ", id: " + router.getId() + ") " + " just switch from " + prevState + " to " + currState; s_logger.info(context); if (currState == RedundantState.MASTER) { _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_DOMAIN_ROUTER, router.getDataCenterId(), router.getPodIdToDeployIn(), title, context); } } } } //Ensure router status is update to date before execute this function. The function would try best to recover all routers except MASTER protected void recoverRedundantNetwork(final DomainRouterVO masterRouter, final DomainRouterVO backupRouter) { if (masterRouter.getState() == State.Running && backupRouter.getState() == State.Running) { final HostVO masterHost = _hostDao.findById(masterRouter.getHostId()); final HostVO backupHost = _hostDao.findById(backupRouter.getHostId()); if (masterHost.getState() == Status.Up && backupHost.getState() == Status.Up) { final String title = "Reboot " + backupRouter.getInstanceName() + " to ensure redundant virtual routers work"; if (s_logger.isDebugEnabled()) { s_logger.debug(title); } _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_DOMAIN_ROUTER, backupRouter.getDataCenterId(), backupRouter.getPodIdToDeployIn(), title, title); try { rebootRouter(backupRouter.getId(), true); } catch (final ConcurrentOperationException e) { s_logger.warn("Fail to reboot " + backupRouter.getInstanceName(), e); } catch (final ResourceUnavailableException e) { s_logger.warn("Fail to reboot " + backupRouter.getInstanceName(), e); } catch (final InsufficientCapacityException e) { s_logger.warn("Fail to reboot " + backupRouter.getInstanceName(), e); } } } } private int getRealPriority(final DomainRouterVO router) { int priority = router.getPriority(); if (router.getIsPriorityBumpUp()) { priority += DEFAULT_DELTA; } return priority; } protected class RvRStatusUpdateTask extends ManagedContextRunnable { public RvRStatusUpdateTask() { } /* * In order to make fail-over works well at any time, we have to ensure: * 1. Backup router's priority = Master's priority - DELTA + 1 * 2. Backup router's priority hasn't been bumped up. */ private void checkSanity(final List routers) { final Set checkedNetwork = new HashSet(); for (final DomainRouterVO router : routers) { if (!router.getIsRedundantRouter()) { continue; } final List routerGuestNtwkIds = _routerDao.getRouterNetworks(router.getId()); for (final Long routerGuestNtwkId : routerGuestNtwkIds) { if (checkedNetwork.contains(routerGuestNtwkId)) { continue; } checkedNetwork.add(routerGuestNtwkId); final List checkingRouters = _routerDao.listByNetworkAndRole(routerGuestNtwkId, Role.VIRTUAL_ROUTER); if (checkingRouters.size() != 2) { continue; } DomainRouterVO masterRouter = null; DomainRouterVO backupRouter = null; for (final DomainRouterVO r : checkingRouters) { if (r.getRedundantState() == RedundantState.MASTER) { if (masterRouter == null) { masterRouter = r; } else { //Duplicate master! We give up, until the admin fix duplicate MASTER issue break; } } else if (r.getRedundantState() == RedundantState.BACKUP) { if (backupRouter == null) { backupRouter = r; } else { break; } } } if (masterRouter != null && backupRouter != null) { if (getRealPriority(masterRouter) - DEFAULT_DELTA + 1 != getRealPriority(backupRouter) || backupRouter.getIsPriorityBumpUp()) { recoverRedundantNetwork(masterRouter, backupRouter); } } } } } private void checkDuplicateMaster(final List routers) { final Map networkRouterMaps = new HashMap(); for (final DomainRouterVO router : routers) { final List routerGuestNtwkIds = _routerDao.getRouterNetworks(router.getId()); for (final Long routerGuestNtwkId : routerGuestNtwkIds) { if (router.getRedundantState() == RedundantState.MASTER) { if (networkRouterMaps.containsKey(routerGuestNtwkId)) { final DomainRouterVO dupRouter = networkRouterMaps.get(routerGuestNtwkId); final String title = "More than one redundant virtual router is in MASTER state! Router " + router.getHostName() + " and router " + dupRouter.getHostName(); final String context = "Virtual router (name: " + router.getHostName() + ", id: " + router.getId() + " and router (name: " + dupRouter.getHostName() + ", id: " + router.getId() + ") are both in MASTER state! If the problem persist, restart both of routers. "; _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_DOMAIN_ROUTER, router.getDataCenterId(), router.getPodIdToDeployIn(), title, context); _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_DOMAIN_ROUTER, dupRouter.getDataCenterId(), dupRouter.getPodIdToDeployIn(), title, context); s_logger.warn(context); } else { networkRouterMaps.put(routerGuestNtwkId, router); } } } } } @Override protected void runInContext() { while (true) { try { final Long networkId = _vrUpdateQueue.take(); // This is a blocking call so this thread won't run all the time if no work item in queue. final List routers = _routerDao.listByNetworkAndRole(networkId, Role.VIRTUAL_ROUTER); if (routers.size() != 2) { continue; } /* * We update the router pair which the lower id router owned by this mgmt server, in order * to prevent duplicate update of router status from cluster mgmt servers */ final DomainRouterVO router0 = routers.get(0); final DomainRouterVO router1 = routers.get(1); DomainRouterVO router = router0; if ((router0.getId() < router1.getId()) && router0.getHostId() != null) { router = router0; } else { router = router1; } if (router.getHostId() == null) { s_logger.debug("Skip router pair (" + router0.getInstanceName() + "," + router1.getInstanceName() + ") due to can't find host"); continue; } final HostVO host = _hostDao.findById(router.getHostId()); if (host == null || host.getManagementServerId() == null || host.getManagementServerId() != ManagementServerNode.getManagementServerId()) { s_logger.debug("Skip router pair (" + router0.getInstanceName() + "," + router1.getInstanceName() + ") due to not belong to this mgmt server"); continue; } updateRoutersRedundantState(routers); checkDuplicateMaster(routers); checkSanity(routers); } catch (final Exception ex) { s_logger.error("Fail to complete the RvRStatusUpdateTask! ", ex); } } } } protected class CheckRouterTask extends ManagedContextRunnable { public CheckRouterTask() { } @Override protected void runInContext() { try { final List routers = _routerDao.listIsolatedByHostId(null); s_logger.debug("Found " + routers.size() + " routers to update status. "); updateSite2SiteVpnConnectionState(routers); final List networks = _networkDao.listRedundantNetworks(); s_logger.debug("Found " + networks.size() + " networks to update RvR status. "); for (final NetworkVO network : networks) { if (!_vrUpdateQueue.offer(network.getId(), 500, TimeUnit.MILLISECONDS)) { s_logger.warn("Cannot insert into virtual router update queue! Adjustment of router.check.interval and router.check.poolsize maybe needed."); break; } } } catch (final Exception ex) { s_logger.error("Fail to complete the CheckRouterTask! ", ex); } } } protected class CheckRouterAlertsTask extends ManagedContextRunnable { public CheckRouterAlertsTask() { } @Override protected void runInContext() { try { getRouterAlerts(); } catch (final Exception ex) { s_logger.error("Fail to complete the CheckRouterAlertsTask! ", ex); } } } protected void getRouterAlerts() { try{ List routers = _routerDao.listByStateAndManagementServer(State.Running, mgmtSrvrId); s_logger.debug("Found " + routers.size() + " running routers. "); for (final DomainRouterVO router : routers) { String serviceMonitoringFlag = SetServiceMonitor.valueIn(router.getDataCenterId()); // Skip the routers in VPC network or skip the routers where Monitor service is not enabled in the corresponding Zone if ( !Boolean.parseBoolean(serviceMonitoringFlag) || router.getVpcId() != null) { continue; } String privateIP = router.getPrivateIpAddress(); if (privateIP != null) { OpRouterMonitorServiceVO opRouterMonitorServiceVO = _opRouterMonitorServiceDao.findById(router.getId()); GetRouterAlertsCommand command = null; if (opRouterMonitorServiceVO == null) { command = new GetRouterAlertsCommand(new String("1970-01-01 00:00:00")); // To avoid sending null value } else { command = new GetRouterAlertsCommand(opRouterMonitorServiceVO.getLastAlertTimestamp()); } command.setAccessDetail(NetworkElementCommand.ROUTER_IP, router.getPrivateIpAddress()); try { final Answer origAnswer = _agentMgr.easySend(router.getHostId(), command); GetRouterAlertsAnswer answer = null; if (origAnswer == null) { s_logger.warn("Unable to get alerts from router " + router.getHostName()); continue; } if (origAnswer instanceof GetRouterAlertsAnswer) { answer = (GetRouterAlertsAnswer)origAnswer; } else { s_logger.warn("Unable to get alerts from router " + router.getHostName()); continue; } if (!answer.getResult()) { s_logger.warn("Unable to get alerts from router " + router.getHostName() + " " + answer.getDetails()); continue; } String alerts[] = answer.getAlerts(); if (alerts != null) { String lastAlertTimeStamp = answer.getTimeStamp(); SimpleDateFormat sdfrmt = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss"); sdfrmt.setLenient(false); try { sdfrmt.parse(lastAlertTimeStamp); } catch (ParseException e) { s_logger.warn("Invalid last alert timestamp received while collecting alerts from router: " + router.getInstanceName()); continue; } for (String alert: alerts) { _alertMgr.sendAlert(AlertType.ALERT_TYPE_DOMAIN_ROUTER, router.getDataCenterId(), router.getPodIdToDeployIn(), "Monitoring Service on VR " + router.getInstanceName(), alert); } if (opRouterMonitorServiceVO == null) { opRouterMonitorServiceVO = new OpRouterMonitorServiceVO(router.getId(), router.getHostName(), lastAlertTimeStamp); _opRouterMonitorServiceDao.persist(opRouterMonitorServiceVO); } else { opRouterMonitorServiceVO.setLastAlertTimestamp(lastAlertTimeStamp); _opRouterMonitorServiceDao.update(opRouterMonitorServiceVO.getId(), opRouterMonitorServiceVO); } } } catch (Exception e) { s_logger.warn("Error while collecting alerts from router: " + router.getInstanceName(), e); continue; } } } } catch (Exception e) { s_logger.warn("Error while collecting alerts from router", e); } } private final static int DEFAULT_PRIORITY = 100; private final static int DEFAULT_DELTA = 2; protected int getUpdatedPriority(final Network guestNetwork, final List routers, final DomainRouterVO exclude) throws InsufficientVirtualNetworkCapcityException { int priority; if (routers.size() == 0) { priority = DEFAULT_PRIORITY; } else { int maxPriority = 0; for (final DomainRouterVO r : routers) { if (!r.getIsRedundantRouter()) { throw new CloudRuntimeException("Redundant router is mixed with single router in one network!"); } //FIXME Assume the maxPriority one should be running or just created. if (r.getId() != exclude.getId() && getRealPriority(r) > maxPriority) { maxPriority = getRealPriority(r); } } if (maxPriority == 0) { return DEFAULT_PRIORITY; } if (maxPriority < 20) { s_logger.error("Current maximum priority is too low!"); throw new InsufficientVirtualNetworkCapcityException("Current maximum priority is too low as " + maxPriority + "!", guestNetwork.getId()); } else if (maxPriority > 200) { s_logger.error("Too many times fail-over happened! Current maximum priority is too high as " + maxPriority + "!"); throw new InsufficientVirtualNetworkCapcityException("Too many times fail-over happened! Current maximum priority is too high as " + maxPriority + "!", guestNetwork.getId()); } priority = maxPriority - DEFAULT_DELTA + 1; } return priority; } /* * Ovm won't support any system. So we have to choose a partner cluster in the same pod to start domain router for us */ private HypervisorType getClusterToStartDomainRouterForOvm(final long podId) { final List clusters = _clusterDao.listByPodId(podId); for (final ClusterVO cv : clusters) { if (cv.getHypervisorType() == HypervisorType.Ovm || cv.getHypervisorType() == HypervisorType.BareMetal) { continue; } final List hosts = _resourceMgr.listAllHostsInCluster(cv.getId()); if (hosts == null || hosts.isEmpty()) { continue; } for (final HostVO h : hosts) { if (h.getState() == Status.Up) { s_logger.debug("Pick up host that has hypervisor type " + h.getHypervisorType() + " in cluster " + cv.getId() + " to start domain router for OVM"); return h.getHypervisorType(); } } } final String errMsg = "Cannot find an available cluster in Pod " + podId + " to start domain router for Ovm. \n Ovm won't support any system vm including domain router, " + "please make sure you have a cluster with hypervisor type of any of xenserver/KVM/Vmware in the same pod" + " with Ovm cluster. And there is at least one host in UP status in that cluster."; throw new CloudRuntimeException(errMsg); } private void checkAndResetPriorityOfRedundantRouter(final List routers) { boolean allStopped = true; for (final DomainRouterVO router : routers) { if (!router.getIsRedundantRouter() || router.getState() != VirtualMachine.State.Stopped) { allStopped = false; break; } } if (!allStopped) { return; } for (final DomainRouterVO router : routers) { // getUpdatedPriority() would update the value later router.setPriority(0); router.setIsPriorityBumpUp(false); _routerDao.update(router.getId(), router); } } @DB protected List findOrDeployVirtualRouterInGuestNetwork(final Network guestNetwork, final DeployDestination dest, Account owner, final boolean isRedundant, final Map params) throws ConcurrentOperationException, InsufficientCapacityException, ResourceUnavailableException { List routers = new ArrayList(); final Network lock = _networkDao.acquireInLockTable(guestNetwork.getId(), NetworkOrchestrationService.NetworkLockTimeout.value()); if (lock == null) { throw new ConcurrentOperationException("Unable to lock network " + guestNetwork.getId()); } if (s_logger.isDebugEnabled()) { s_logger.debug("Lock is acquired for network id " + lock.getId() + " as a part of router startup in " + dest); } try { assert guestNetwork.getState() == Network.State.Implemented || guestNetwork.getState() == Network.State.Setup || guestNetwork.getState() == Network.State.Implementing : "Network is not yet fully implemented: " + guestNetwork; assert guestNetwork.getTrafficType() == TrafficType.Guest; // 1) Get deployment plan and find out the list of routers final boolean isPodBased = (dest.getDataCenter().getNetworkType() == NetworkType.Basic); // dest has pod=null, for Basic Zone findOrDeployVRs for all Pods final List destinations = new ArrayList(); // for basic zone, if 'dest' has pod set to null then this is network restart scenario otherwise it is a vm deployment scenario if (dest.getDataCenter().getNetworkType() == NetworkType.Basic && dest.getPod() == null) { // Find all pods in the data center with running or starting user vms final long dcId = dest.getDataCenter().getId(); final List pods = listByDataCenterIdVMTypeAndStates(dcId, VirtualMachine.Type.User, VirtualMachine.State.Starting, VirtualMachine.State.Running); // Loop through all the pods skip those with running or starting VRs for (final HostPodVO pod : pods) { // Get list of VRs in starting or running state final long podId = pod.getId(); final List virtualRouters = _routerDao.listByPodIdAndStates(podId, VirtualMachine.State.Starting, VirtualMachine.State.Running); assert (virtualRouters.size() <= 1) : "Pod can have utmost one VR in Basic Zone, please check!"; // Add virtualRouters to the routers, this avoids the situation when // all routers are skipped and VirtualRouterElement throws exception routers.addAll(virtualRouters); // If List size is one, we already have a starting or running VR, skip deployment if (virtualRouters.size() == 1) { s_logger.debug("Skipping VR deployment: Found a running or starting VR in Pod " + pod.getName() + " id=" + podId); continue; } // Add new DeployDestination for this pod destinations.add(new DeployDestination(dest.getDataCenter(), pod, null, null)); } } else { // Else, just add the supplied dest destinations.add(dest); } // Except for Basic Zone, the for loop will iterate only once for (final DeployDestination destination : destinations) { final Pair> planAndRouters = getDeploymentPlanAndRouters(isPodBased, destination, guestNetwork.getId()); routers = planAndRouters.second(); // 2) Figure out required routers count int routerCount = 1; if (isRedundant) { routerCount = 2; //Check current redundant routers, if possible(all routers are stopped), reset the priority if (routers.size() != 0) { checkAndResetPriorityOfRedundantRouter(routers); } } // If old network is redundant but new is single router, then routers.size() = 2 but routerCount = 1 if (routers.size() >= routerCount) { return routers; } if (routers.size() >= 5) { s_logger.error("Too much redundant routers!"); } // Check if providers are supported in the physical networks final Type type = Type.VirtualRouter; final Long physicalNetworkId = _networkModel.getPhysicalNetworkId(guestNetwork); final PhysicalNetworkServiceProvider provider = _physicalProviderDao.findByServiceProvider(physicalNetworkId, type.toString()); if (provider == null) { throw new CloudRuntimeException("Cannot find service provider " + type.toString() + " in physical network " + physicalNetworkId); } final VirtualRouterProvider vrProvider = _vrProviderDao.findByNspIdAndType(provider.getId(), type); if (vrProvider == null) { throw new CloudRuntimeException("Cannot find virtual router provider " + type.toString() + " as service provider " + provider.getId()); } if (_networkModel.isNetworkSystem(guestNetwork) || guestNetwork.getGuestType() == Network.GuestType.Shared) { owner = _accountMgr.getAccount(Account.ACCOUNT_ID_SYSTEM); } // Check if public network has to be set on VR boolean publicNetwork = false; if (_networkModel.isProviderSupportServiceInNetwork(guestNetwork.getId(), Service.SourceNat, Provider.VirtualRouter)) { publicNetwork = true; } if (isRedundant && !publicNetwork) { s_logger.error("Didn't support redundant virtual router without public network!"); return null; } Long offeringId = _networkOfferingDao.findById(guestNetwork.getNetworkOfferingId()).getServiceOfferingId(); if (offeringId == null) { offeringId = _offering.getId(); } PublicIp sourceNatIp = null; if (publicNetwork) { sourceNatIp = _ipAddrMgr.assignSourceNatIpAddressToGuestNetwork(owner, guestNetwork); } // 3) deploy virtual router(s) final int count = routerCount - routers.size(); final DeploymentPlan plan = planAndRouters.first(); for (int i = 0; i < count; i++) { LinkedHashMap> networks = createRouterNetworks(owner, isRedundant, plan, guestNetwork, new Pair( publicNetwork, sourceNatIp)); //don't start the router as we are holding the network lock that needs to be released at the end of router allocation DomainRouterVO router = deployRouter(owner, destination, plan, params, isRedundant, vrProvider, offeringId, null, networks, false, null); if (router != null) { _routerDao.addRouterToGuestNetwork(router, guestNetwork); routers.add(router); } } } } finally { if (lock != null) { _networkDao.releaseFromLockTable(lock.getId()); if (s_logger.isDebugEnabled()) { s_logger.debug("Lock is released for network id " + lock.getId() + " as a part of router startup in " + dest); } } } return routers; } protected List listByDataCenterIdVMTypeAndStates(final long id, final VirtualMachine.Type type, final VirtualMachine.State... states) { final SearchBuilder vmInstanceSearch = _vmDao.createSearchBuilder(); vmInstanceSearch.and("type", vmInstanceSearch.entity().getType(), SearchCriteria.Op.EQ); vmInstanceSearch.and("states", vmInstanceSearch.entity().getState(), SearchCriteria.Op.IN); final SearchBuilder podIdSearch = _podDao.createSearchBuilder(); podIdSearch.and("dc", podIdSearch.entity().getDataCenterId(), SearchCriteria.Op.EQ); podIdSearch.select(null, SearchCriteria.Func.DISTINCT, podIdSearch.entity().getId()); podIdSearch.join("vmInstanceSearch", vmInstanceSearch, podIdSearch.entity().getId(), vmInstanceSearch.entity().getPodIdToDeployIn(), JoinBuilder.JoinType.INNER); podIdSearch.done(); final SearchCriteria sc = podIdSearch.create(); sc.setParameters("dc", id); sc.setJoinParameters("vmInstanceSearch", "type", type); sc.setJoinParameters("vmInstanceSearch", "states", (Object[])states); return _podDao.search(sc, null); } protected DomainRouterVO deployRouter(final Account owner, final DeployDestination dest, final DeploymentPlan plan, final Map params, final boolean isRedundant, final VirtualRouterProvider vrProvider, final long svcOffId, final Long vpcId, final LinkedHashMap> networks, final boolean startRouter, final List supportedHypervisors) throws ConcurrentOperationException, InsufficientAddressCapacityException, InsufficientServerCapacityException, InsufficientCapacityException, StorageUnavailableException, ResourceUnavailableException { final ServiceOfferingVO routerOffering = _serviceOfferingDao.findById(svcOffId); // Router is the network element, we don't know the hypervisor type yet. // Try to allocate the domR twice using diff hypervisors, and when failed both times, throw the exception up final List hypervisors = getHypervisors(dest, plan, supportedHypervisors); int allocateRetry = 0; int startRetry = 0; DomainRouterVO router = null; for (final Iterator iter = hypervisors.iterator(); iter.hasNext();) { final HypervisorType hType = iter.next(); try { final long id = _routerDao.getNextInSequence(Long.class, "id"); if (s_logger.isDebugEnabled()) { s_logger.debug("Allocating the VR i=" + id + " in datacenter " + dest.getDataCenter() + "with the hypervisor type " + hType); } String templateName = null; switch (hType) { case XenServer: templateName = RouterTemplateXen.valueIn(dest.getDataCenter().getId()); break; case KVM: templateName = RouterTemplateKvm.valueIn(dest.getDataCenter().getId()); break; case VMware: templateName = RouterTemplateVmware.valueIn(dest.getDataCenter().getId()); break; case Hyperv: templateName = RouterTemplateHyperV.valueIn(dest.getDataCenter().getId()); break; case LXC: templateName = RouterTemplateLxc.valueIn(dest.getDataCenter().getId()); break; default: break; } final VMTemplateVO template = _templateDao.findRoutingTemplate(hType, templateName); if (template == null) { s_logger.debug(hType + " won't support system vm, skip it"); continue; } boolean offerHA = routerOffering.getOfferHA(); /* We don't provide HA to redundant router VMs, admin should own it all, and redundant router themselves are HA */ if (isRedundant) { offerHA = false; } router = new DomainRouterVO(id, routerOffering.getId(), vrProvider.getId(), VirtualMachineName.getRouterName(id, _instance), template.getId(), template.getHypervisorType(), template.getGuestOSId(), owner.getDomainId(), owner.getId(), isRedundant, 0, false, RedundantState.UNKNOWN, offerHA, false, vpcId); router.setDynamicallyScalable(template.isDynamicallyScalable()); router.setRole(Role.VIRTUAL_ROUTER); router = _routerDao.persist(router); _itMgr.allocate(router.getInstanceName(), template, routerOffering, networks, plan, null); router = _routerDao.findById(router.getId()); } catch (final InsufficientCapacityException ex) { if (allocateRetry < 2 && iter.hasNext()) { s_logger.debug("Failed to allocate the VR with hypervisor type " + hType + ", retrying one more time"); continue; } else { throw ex; } } finally { allocateRetry++; } if (startRouter) { try { router = startVirtualRouter(router, _accountMgr.getSystemUser(), _accountMgr.getSystemAccount(), params); break; } catch (final InsufficientCapacityException ex) { if (startRetry < 2 && iter.hasNext()) { s_logger.debug("Failed to start the VR " + router + " with hypervisor type " + hType + ", " + "destroying it and recreating one more time"); // destroy the router destroyRouter(router.getId(), _accountMgr.getAccount(Account.ACCOUNT_ID_SYSTEM), User.UID_SYSTEM); continue; } else { throw ex; } } finally { startRetry++; } } else { //return stopped router return router; } } return router; } protected List getHypervisors(final DeployDestination dest, final DeploymentPlan plan, final List supportedHypervisors) throws InsufficientServerCapacityException { List hypervisors = new ArrayList(); if (dest.getCluster() != null) { if (dest.getCluster().getHypervisorType() == HypervisorType.Ovm) { hypervisors.add(getClusterToStartDomainRouterForOvm(dest.getCluster().getPodId())); } else { hypervisors.add(dest.getCluster().getHypervisorType()); } } else { final HypervisorType defaults = _resourceMgr.getDefaultHypervisor(dest.getDataCenter().getId()); if (defaults != HypervisorType.None) { hypervisors.add(defaults); } else { //if there is no default hypervisor, get it from the cluster hypervisors = _resourceMgr.getSupportedHypervisorTypes(dest.getDataCenter().getId(), true, plan.getPodId()); } } //keep only elements defined in supported hypervisors final StringBuilder hTypesStr = new StringBuilder(); if (supportedHypervisors != null && !supportedHypervisors.isEmpty()) { hypervisors.retainAll(supportedHypervisors); for (final HypervisorType hType : supportedHypervisors) { hTypesStr.append(hType).append(" "); } } if (hypervisors.isEmpty()) { final String errMsg = (hTypesStr.capacity() > 0) ? "supporting hypervisors " + hTypesStr.toString() : ""; if (plan.getPodId() != null) { throw new InsufficientServerCapacityException("Unable to create virtual router, " + "there are no clusters in the pod " + errMsg, Pod.class, plan.getPodId()); } throw new InsufficientServerCapacityException("Unable to create virtual router, " + "there are no clusters in the zone " + errMsg, DataCenter.class, dest.getDataCenter().getId()); } return hypervisors; } protected LinkedHashMap> createRouterNetworks(final Account owner, final boolean isRedundant, final DeploymentPlan plan, final Network guestNetwork, final Pair publicNetwork) throws ConcurrentOperationException, InsufficientAddressCapacityException { boolean setupPublicNetwork = false; if (publicNetwork != null) { setupPublicNetwork = publicNetwork.first(); } //Form networks LinkedHashMap> networks = new LinkedHashMap>(3); //1) Guest network boolean hasGuestNetwork = false; if (guestNetwork != null) { s_logger.debug("Adding nic for Virtual Router in Guest network " + guestNetwork); String defaultNetworkStartIp = null, defaultNetworkStartIpv6 = null; if (!setupPublicNetwork) { final Nic placeholder = _networkModel.getPlaceholderNicForRouter(guestNetwork, plan.getPodId()); if (guestNetwork.getCidr() != null) { if (placeholder != null && placeholder.getIp4Address() != null) { s_logger.debug("Requesting ipv4 address " + placeholder.getIp4Address() + " stored in placeholder nic for the network " + guestNetwork); defaultNetworkStartIp = placeholder.getIp4Address(); } else { final String startIp = _networkModel.getStartIpAddress(guestNetwork.getId()); if (startIp != null && _ipAddressDao.findByIpAndSourceNetworkId(guestNetwork.getId(), startIp).getAllocatedTime() == null) { defaultNetworkStartIp = startIp; } else if (s_logger.isDebugEnabled()) { s_logger.debug("First ipv4 " + startIp + " in network id=" + guestNetwork.getId() + " is already allocated, can't use it for domain router; will get random ip address from the range"); } } } if (guestNetwork.getIp6Cidr() != null) { if (placeholder != null && placeholder.getIp6Address() != null) { s_logger.debug("Requesting ipv6 address " + placeholder.getIp6Address() + " stored in placeholder nic for the network " + guestNetwork); defaultNetworkStartIpv6 = placeholder.getIp6Address(); } else { final String startIpv6 = _networkModel.getStartIpv6Address(guestNetwork.getId()); if (startIpv6 != null && _ipv6Dao.findByNetworkIdAndIp(guestNetwork.getId(), startIpv6) == null) { defaultNetworkStartIpv6 = startIpv6; } else if (s_logger.isDebugEnabled()) { s_logger.debug("First ipv6 " + startIpv6 + " in network id=" + guestNetwork.getId() + " is already allocated, can't use it for domain router; will get random ipv6 address from the range"); } } } } final NicProfile gatewayNic = new NicProfile(defaultNetworkStartIp, defaultNetworkStartIpv6); if (setupPublicNetwork) { if (isRedundant) { gatewayNic.setIp4Address(_ipAddrMgr.acquireGuestIpAddress(guestNetwork, null)); } else { gatewayNic.setIp4Address(guestNetwork.getGateway()); } gatewayNic.setBroadcastUri(guestNetwork.getBroadcastUri()); gatewayNic.setBroadcastType(guestNetwork.getBroadcastDomainType()); gatewayNic.setIsolationUri(guestNetwork.getBroadcastUri()); gatewayNic.setMode(guestNetwork.getMode()); final String gatewayCidr = guestNetwork.getCidr(); gatewayNic.setNetmask(NetUtils.getCidrNetmask(gatewayCidr)); } else { gatewayNic.setDefaultNic(true); } networks.put(guestNetwork, new ArrayList(Arrays.asList(gatewayNic))); hasGuestNetwork = true; } //2) Control network s_logger.debug("Adding nic for Virtual Router in Control network "); List offerings = _networkModel.getSystemAccountNetworkOfferings(NetworkOffering.SystemControlNetwork); NetworkOffering controlOffering = offerings.get(0); Network controlConfig = _networkMgr.setupNetwork(_systemAcct, controlOffering, plan, null, null, false).get(0); networks.put(controlConfig, new ArrayList()); //3) Public network if (setupPublicNetwork) { final PublicIp sourceNatIp = publicNetwork.second(); s_logger.debug("Adding nic for Virtual Router in Public network "); //if source nat service is supported by the network, get the source nat ip address final NicProfile defaultNic = new NicProfile(); defaultNic.setDefaultNic(true); defaultNic.setIp4Address(sourceNatIp.getAddress().addr()); defaultNic.setGateway(sourceNatIp.getGateway()); defaultNic.setNetmask(sourceNatIp.getNetmask()); defaultNic.setMacAddress(sourceNatIp.getMacAddress()); // get broadcast from public network final Network pubNet = _networkDao.findById(sourceNatIp.getNetworkId()); if (pubNet.getBroadcastDomainType() == BroadcastDomainType.Vxlan) { defaultNic.setBroadcastType(BroadcastDomainType.Vxlan); defaultNic.setBroadcastUri(BroadcastDomainType.Vxlan.toUri(sourceNatIp.getVlanTag())); defaultNic.setIsolationUri(BroadcastDomainType.Vxlan.toUri(sourceNatIp.getVlanTag())); } else { defaultNic.setBroadcastType(BroadcastDomainType.Vlan); defaultNic.setBroadcastUri(BroadcastDomainType.Vlan.toUri(sourceNatIp.getVlanTag())); defaultNic.setIsolationUri(IsolationType.Vlan.toUri(sourceNatIp.getVlanTag())); } if (hasGuestNetwork) { defaultNic.setDeviceId(2); } final NetworkOffering publicOffering = _networkModel.getSystemAccountNetworkOfferings(NetworkOffering.SystemPublicNetwork).get(0); final List publicNetworks = _networkMgr.setupNetwork(_systemAcct, publicOffering, plan, null, null, false); final String publicIp = defaultNic.getIp4Address(); // We want to use the identical MAC address for RvR on public interface if possible final NicVO peerNic = _nicDao.findByIp4AddressAndNetworkId(publicIp, publicNetworks.get(0).getId()); if (peerNic != null) { s_logger.info("Use same MAC as previous RvR, the MAC is " + peerNic.getMacAddress()); defaultNic.setMacAddress(peerNic.getMacAddress()); } networks.put(publicNetworks.get(0), new ArrayList(Arrays.asList(defaultNic))); } return networks; } protected Pair> getDeploymentPlanAndRouters(final boolean isPodBased, final DeployDestination dest, final long guestNetworkId) { final long dcId = dest.getDataCenter().getId(); List routers = null; DeploymentPlan plan = new DataCenterDeployment(dcId); if (isPodBased) { final Pod pod = dest.getPod(); Long podId = null; if (pod != null) { podId = pod.getId(); } else { throw new CloudRuntimeException("Pod id is expected in deployment destination"); } routers = _routerDao.listByNetworkAndPodAndRole(guestNetworkId, podId, Role.VIRTUAL_ROUTER); plan = new DataCenterDeployment(dcId, podId, null, null, null, null); } else { routers = _routerDao.listByNetworkAndRole(guestNetworkId, Role.VIRTUAL_ROUTER); } return new Pair>(plan, routers); } private DomainRouterVO waitRouter(DomainRouterVO router) { DomainRouterVO vm = _routerDao.findById(router.getId()); if (s_logger.isDebugEnabled()) s_logger.debug("Router " + router.getInstanceName() + " is not fully up yet, we will wait"); while (vm.getState() == State.Starting) { try { Thread.sleep(1000); } catch (InterruptedException e) { } // reload to get the latest state info vm = _routerDao.findById(router.getId()); } if (vm.getState() == State.Running) { if (s_logger.isDebugEnabled()) s_logger.debug("Router " + router.getInstanceName() + " is now fully up"); return router; } s_logger.warn("Router " + router.getInstanceName() + " failed to start. current state: " + vm.getState()); return null; } private DomainRouterVO startVirtualRouter(DomainRouterVO router, User user, Account caller, Map params) throws StorageUnavailableException, InsufficientCapacityException, ConcurrentOperationException, ResourceUnavailableException { if (router.getRole() != Role.VIRTUAL_ROUTER || !router.getIsRedundantRouter()) { return this.start(router, user, caller, params, null); } if (router.getState() == State.Running) { s_logger.debug("Redundant router " + router.getInstanceName() + " is already running!"); return router; } // // If another thread has already requested a VR start, there is a transition period for VR to transit from // Starting to Running, there exist a race conditioning window here // We will wait until VR is up or fail if (router.getState() == State.Starting) { return waitRouter(router); } DataCenterDeployment plan = new DataCenterDeployment(0, null, null, null, null, null); DomainRouterVO result = null; assert router.getIsRedundantRouter(); final List networkIds = _routerDao.getRouterNetworks(router.getId()); //Not support VPC now if (networkIds.size() > 1) { throw new ResourceUnavailableException("Unable to support more than one guest network for redundant router now!", DataCenter.class, router.getDataCenterId()); } DomainRouterVO routerToBeAvoid = null; if (networkIds.size() != 0) { final List routerList = _routerDao.findByNetwork(networkIds.get(0)); for (final DomainRouterVO rrouter : routerList) { if (rrouter.getHostId() != null && rrouter.getIsRedundantRouter() && rrouter.getState() == State.Running) { if (routerToBeAvoid != null) { throw new ResourceUnavailableException("Try to start router " + router.getInstanceName() + "(" + router.getId() + ")" + ", but there are already two redundant routers with IP " + router.getPublicIpAddress() + ", they are " + rrouter.getInstanceName() + "(" + rrouter.getId() + ") and " + routerToBeAvoid.getInstanceName() + "(" + routerToBeAvoid.getId() + ")", DataCenter.class, rrouter.getDataCenterId()); } routerToBeAvoid = rrouter; } } } if (routerToBeAvoid == null) { return this.start(router, user, caller, params, null); } // We would try best to deploy the router to another place final int retryIndex = 5; final ExcludeList[] avoids = new ExcludeList[5]; avoids[0] = new ExcludeList(); avoids[0].addPod(routerToBeAvoid.getPodIdToDeployIn()); avoids[1] = new ExcludeList(); avoids[1].addCluster(_hostDao.findById(routerToBeAvoid.getHostId()).getClusterId()); avoids[2] = new ExcludeList(); final List volumes = _volumeDao.findByInstanceAndType(routerToBeAvoid.getId(), Volume.Type.ROOT); if (volumes != null && volumes.size() != 0) { avoids[2].addPool(volumes.get(0).getPoolId()); } avoids[2].addHost(routerToBeAvoid.getHostId()); avoids[3] = new ExcludeList(); avoids[3].addHost(routerToBeAvoid.getHostId()); avoids[4] = new ExcludeList(); for (int i = 0; i < retryIndex; i++) { if (s_logger.isTraceEnabled()) { s_logger.trace("Try to deploy redundant virtual router:" + router.getHostName() + ", for " + i + " time"); } plan.setAvoids(avoids[i]); try { result = this.start(router, user, caller, params, plan); } catch (final InsufficientServerCapacityException ex) { result = null; } if (result != null) { break; } } return result; } @Override public List deployVirtualRouterInGuestNetwork(final Network guestNetwork, final DeployDestination dest, final Account owner, final Map params, final boolean isRedundant) throws InsufficientCapacityException, ConcurrentOperationException, ResourceUnavailableException { final List routers = findOrDeployVirtualRouterInGuestNetwork(guestNetwork, dest, owner, isRedundant, params); return startRouters(params, routers); } protected List startRouters(final Map params, final List routers) throws StorageUnavailableException, InsufficientCapacityException, ConcurrentOperationException, ResourceUnavailableException { List runningRouters = null; if (routers != null) { runningRouters = new ArrayList(); } for (DomainRouterVO router : routers) { boolean skip = false; final State state = router.getState(); if (router.getHostId() != null && state != State.Running) { final HostVO host = _hostDao.findById(router.getHostId()); if (host == null || host.getState() != Status.Up) { skip = true; } } if (!skip) { if (state != State.Running) { router = startVirtualRouter(router, _accountMgr.getSystemUser(), _accountMgr.getSystemAccount(), params); } if (router != null) { runningRouters.add(router); } } } return runningRouters; } @Override public boolean finalizeVirtualMachineProfile(final VirtualMachineProfile profile, final DeployDestination dest, final ReservationContext context) { boolean dnsProvided = true; boolean dhcpProvided = true; boolean publicNetwork = false; final DataCenterVO dc = _dcDao.findById(dest.getDataCenter().getId()); _dcDao.loadDetails(dc); //1) Set router details final DomainRouterVO router = _routerDao.findById(profile.getVirtualMachine().getId()); final Map details = _vmDetailsDao.listDetailsKeyPairs(router.getId()); router.setDetails(details); //2) Prepare boot loader elements related with Control network final StringBuilder buf = profile.getBootArgsBuilder(); buf.append(" template=domP"); buf.append(" name=").append(profile.getHostName()); if (Boolean.valueOf(_configDao.getValue("system.vm.random.password"))) { buf.append(" vmpassword=").append(_configDao.getValue("system.vm.password")); } NicProfile controlNic = null; String defaultDns1 = null; String defaultDns2 = null; String defaultIp6Dns1 = null; String defaultIp6Dns2 = null; for (final NicProfile nic : profile.getNics()) { final int deviceId = nic.getDeviceId(); boolean ipv4 = false, ipv6 = false; if (nic.getIp4Address() != null) { ipv4 = true; buf.append(" eth").append(deviceId).append("ip=").append(nic.getIp4Address()); buf.append(" eth").append(deviceId).append("mask=").append(nic.getNetmask()); } if (nic.getIp6Address() != null) { ipv6 = true; buf.append(" eth").append(deviceId).append("ip6=").append(nic.getIp6Address()); buf.append(" eth").append(deviceId).append("ip6prelen=").append(NetUtils.getIp6CidrSize(nic.getIp6Cidr())); } if (nic.isDefaultNic()) { if (ipv4) { buf.append(" gateway=").append(nic.getGateway()); } if (ipv6) { buf.append(" ip6gateway=").append(nic.getIp6Gateway()); } defaultDns1 = nic.getDns1(); defaultDns2 = nic.getDns2(); defaultIp6Dns1 = nic.getIp6Dns1(); defaultIp6Dns2 = nic.getIp6Dns2(); } if (nic.getTrafficType() == TrafficType.Management) { buf.append(" localgw=").append(dest.getPod().getGateway()); } else if (nic.getTrafficType() == TrafficType.Control) { controlNic = nic; // DOMR control command is sent over management server in VMware if (dest.getHost().getHypervisorType() == HypervisorType.VMware || dest.getHost().getHypervisorType() == HypervisorType.Hyperv) { s_logger.info("Check if we need to add management server explicit route to DomR. pod cidr: " + dest.getPod().getCidrAddress() + "/" + dest.getPod().getCidrSize() + ", pod gateway: " + dest.getPod().getGateway() + ", management host: " + ApiServiceConfiguration.ManagementHostIPAdr.value()); if (s_logger.isInfoEnabled()) { s_logger.info("Add management server explicit route to DomR."); } // always add management explicit route, for basic networking setup, DomR may have two interfaces while both // are on the same subnet _mgmtCidr = _configDao.getValue(Config.ManagementNetwork.key()); if (NetUtils.isValidCIDR(_mgmtCidr)) { buf.append(" mgmtcidr=").append(_mgmtCidr); buf.append(" localgw=").append(dest.getPod().getGateway()); } if (dc.getNetworkType() == NetworkType.Basic) { // ask domR to setup SSH on guest network buf.append(" sshonguest=true"); } } } else if (nic.getTrafficType() == TrafficType.Guest) { dnsProvided = _networkModel.isProviderSupportServiceInNetwork(nic.getNetworkId(), Service.Dns, Provider.VirtualRouter); dhcpProvided = _networkModel.isProviderSupportServiceInNetwork(nic.getNetworkId(), Service.Dhcp, Provider.VirtualRouter); //build bootloader parameter for the guest buf.append(createGuestBootLoadArgs(nic, defaultDns1, defaultDns2, router)); } else if (nic.getTrafficType() == TrafficType.Public) { publicNetwork = true; } } if (controlNic == null) { throw new CloudRuntimeException("Didn't start a control port"); } final String rpValue = _configDao.getValue(Config.NetworkRouterRpFilter.key()); if (rpValue != null && rpValue.equalsIgnoreCase("true")) { _disableRpFilter = true; } else { _disableRpFilter = false; } String rpFilter = " "; String type = null; if (router.getVpcId() != null) { type = "vpcrouter"; if (_disableRpFilter) { rpFilter = " disable_rp_filter=true"; } } else if (!publicNetwork) { type = "dhcpsrvr"; } else { type = "router"; if (_disableRpFilter) { rpFilter = " disable_rp_filter=true"; } } if (_disableRpFilter) { rpFilter = " disable_rp_filter=true"; } buf.append(" type=" + type + rpFilter); final String domain_suffix = dc.getDetail(ZoneConfig.DnsSearchOrder.getName()); if (domain_suffix != null) { buf.append(" dnssearchorder=").append(domain_suffix); } if (profile.getHypervisorType() == HypervisorType.VMware || profile.getHypervisorType() == HypervisorType.Hyperv) { buf.append(" extra_pubnics=" + _routerExtraPublicNics); } /* If virtual router didn't provide DNS service but provide DHCP service, we need to override the DHCP response * to return DNS server rather than * virtual router itself. */ if (dnsProvided || dhcpProvided) { if (defaultDns1 != null) { buf.append(" dns1=").append(defaultDns1); } if (defaultDns2 != null) { buf.append(" dns2=").append(defaultDns2); } if (defaultIp6Dns1 != null) { buf.append(" ip6dns1=").append(defaultIp6Dns1); } if (defaultIp6Dns2 != null) { buf.append(" ip6dns2=").append(defaultIp6Dns2); } boolean useExtDns = !dnsProvided; /* For backward compatibility */ useExtDns = useExtDns || UseExternalDnsServers.valueIn(dc.getId()); if (useExtDns) { buf.append(" useextdns=true"); } } if (s_logger.isDebugEnabled()) { s_logger.debug("Boot Args for " + profile + ": " + buf.toString()); } return true; } protected StringBuilder createGuestBootLoadArgs(final NicProfile guestNic, final String defaultDns1, final String defaultDns2, DomainRouterVO router) { final long guestNetworkId = guestNic.getNetworkId(); final NetworkVO guestNetwork = _networkDao.findById(guestNetworkId); String dhcpRange = null; final DataCenterVO dc = _dcDao.findById(guestNetwork.getDataCenterId()); final StringBuilder buf = new StringBuilder(); final boolean isRedundant = router.getIsRedundantRouter(); if (isRedundant) { buf.append(" redundant_router=1"); final List routers = _routerDao.listByNetworkAndRole(guestNetwork.getId(), Role.VIRTUAL_ROUTER); try { final int priority = getUpdatedPriority(guestNetwork, routers, router); router.setPriority(priority); router = _routerDao.persist(router); } catch (final InsufficientVirtualNetworkCapcityException e) { s_logger.error("Failed to get update priority!", e); throw new CloudRuntimeException("Failed to get update priority!"); } final Network net = _networkModel.getNetwork(guestNic.getNetworkId()); buf.append(" guestgw=").append(net.getGateway()); final String brd = NetUtils.long2Ip(NetUtils.ip2Long(guestNic.getIp4Address()) | ~NetUtils.ip2Long(guestNic.getNetmask())); buf.append(" guestbrd=").append(brd); buf.append(" guestcidrsize=").append(NetUtils.getCidrSize(guestNic.getNetmask())); buf.append(" router_pr=").append(router.getPriority()); int advertInt = NumbersUtil.parseInt(_configDao.getValue(Config.RedundantRouterVrrpInterval.key()), 1); buf.append(" advert_int=").append(advertInt); } //setup network domain final String domain = guestNetwork.getNetworkDomain(); if (domain != null) { buf.append(" domain=" + domain); } long cidrSize = 0; //setup dhcp range if (dc.getNetworkType() == NetworkType.Basic) { if (guestNic.isDefaultNic()) { cidrSize = NetUtils.getCidrSize(guestNic.getNetmask()); final String cidr = NetUtils.getCidrSubNet(guestNic.getGateway(), cidrSize); if (cidr != null) { dhcpRange = NetUtils.getIpRangeStartIpFromCidr(cidr, cidrSize); } } } else if (dc.getNetworkType() == NetworkType.Advanced) { final String cidr = guestNetwork.getCidr(); if (cidr != null) { cidrSize = NetUtils.getCidrSize(NetUtils.getCidrNetmask(cidr)); dhcpRange = NetUtils.getDhcpRange(cidr); } } if (dhcpRange != null) { // To limit DNS to the cidr range buf.append(" cidrsize=" + String.valueOf(cidrSize)); buf.append(" dhcprange=" + dhcpRange); } return buf; } protected String getGuestDhcpRange(final NicProfile guestNic, final Network guestNetwork, final DataCenter dc) { String dhcpRange = null; //setup dhcp range if (dc.getNetworkType() == NetworkType.Basic) { final long cidrSize = NetUtils.getCidrSize(guestNic.getNetmask()); final String cidr = NetUtils.getCidrSubNet(guestNic.getGateway(), cidrSize); if (cidr != null) { dhcpRange = NetUtils.getIpRangeStartIpFromCidr(cidr, cidrSize); } } else if (dc.getNetworkType() == NetworkType.Advanced) { final String cidr = guestNetwork.getCidr(); if (cidr != null) { dhcpRange = NetUtils.getDhcpRange(cidr); } } return dhcpRange; } @Override public boolean setupDhcpForPvlan(final boolean add, final DomainRouterVO router, final Long hostId, final NicProfile nic) { if (!nic.getBroadCastUri().getScheme().equals("pvlan")) { return false; } String op = "add"; if (!add) { op = "delete"; } final Network network = _networkDao.findById(nic.getNetworkId()); final String networkTag = _networkModel.getNetworkTag(router.getHypervisorType(), network); final PvlanSetupCommand cmd = PvlanSetupCommand.createDhcpSetup(op, nic.getBroadCastUri(), networkTag, router.getInstanceName(), nic.getMacAddress(), nic.getIp4Address()); // In fact we send command to the host of router, we're not programming router but the host Answer answer = null; try { answer = _agentMgr.send(hostId, cmd); } catch (final OperationTimedoutException e) { s_logger.warn("Timed Out", e); return false; } catch (final AgentUnavailableException e) { s_logger.warn("Agent Unavailable ", e); return false; } if (answer == null || !answer.getResult()) { return false; } return true; } @Override public boolean finalizeDeployment(final Commands cmds, final VirtualMachineProfile profile, final DeployDestination dest, final ReservationContext context) throws ResourceUnavailableException { final DomainRouterVO router = _routerDao.findById(profile.getId()); final List nics = profile.getNics(); for (final NicProfile nic : nics) { if (nic.getTrafficType() == TrafficType.Public) { router.setPublicIpAddress(nic.getIp4Address()); router.setPublicNetmask(nic.getNetmask()); router.setPublicMacAddress(nic.getMacAddress()); } else if (nic.getTrafficType() == TrafficType.Control) { router.setPrivateIpAddress(nic.getIp4Address()); router.setPrivateMacAddress(nic.getMacAddress()); } } _routerDao.update(router.getId(), router); finalizeCommandsOnStart(cmds, profile); return true; } @Override public boolean finalizeCommandsOnStart(final Commands cmds, final VirtualMachineProfile profile) { final DomainRouterVO router = _routerDao.findById(profile.getId()); final NicProfile controlNic = getControlNic(profile); if (controlNic == null) { s_logger.error("Control network doesn't exist for the router " + router); return false; } finalizeSshAndVersionAndNetworkUsageOnStart(cmds, profile, router, controlNic); // restart network if restartNetwork = false is not specified in profile parameters boolean reprogramGuestNtwks = true; if (profile.getParameter(Param.ReProgramGuestNetworks) != null && (Boolean)profile.getParameter(Param.ReProgramGuestNetworks) == false) { reprogramGuestNtwks = false; } final VirtualRouterProvider vrProvider = _vrProviderDao.findById(router.getElementId()); if (vrProvider == null) { throw new CloudRuntimeException("Cannot find related virtual router provider of router: " + router.getHostName()); } final Provider provider = Network.Provider.getProvider(vrProvider.getType().toString()); if (provider == null) { throw new CloudRuntimeException("Cannot find related provider of virtual router provider: " + vrProvider.getType().toString()); } final List routerGuestNtwkIds = _routerDao.getRouterNetworks(router.getId()); for (final Long guestNetworkId : routerGuestNtwkIds) { AggregationControlCommand startCmd = new AggregationControlCommand(Action.Start, router.getInstanceName(), controlNic.getIp4Address(), getRouterIpInNetwork(guestNetworkId, router.getId())); cmds.addCommand(startCmd); if (reprogramGuestNtwks) { finalizeIpAssocForNetwork(cmds, router, provider, guestNetworkId, null); finalizeNetworkRulesForNetwork(cmds, router, provider, guestNetworkId); NetworkOffering offering = _networkOfferingDao.findById((_networkDao.findById(guestNetworkId)).getNetworkOfferingId()); //service monitoring is currently not added in RVR if (!offering.getRedundantRouter()) { String serviceMonitringSet = _configDao.getValue(Config.EnableServiceMonitoring.key()); if (serviceMonitringSet != null && serviceMonitringSet.equalsIgnoreCase("true")) { finalizeMonitorServiceOnStrat(cmds, profile, router, provider, guestNetworkId, true); } else { finalizeMonitorServiceOnStrat(cmds, profile, router, provider, guestNetworkId, false); } } } finalizeUserDataAndDhcpOnStart(cmds, router, provider, guestNetworkId); AggregationControlCommand finishCmd = new AggregationControlCommand(Action.Finish, router.getInstanceName(), controlNic.getIp4Address(), getRouterIpInNetwork(guestNetworkId, router.getId())); cmds.addCommand(finishCmd); } return true; } private void finalizeMonitorServiceOnStrat(Commands cmds, VirtualMachineProfile profile, DomainRouterVO router, Provider provider, long networkId, Boolean add) { final NetworkVO network = _networkDao.findById(networkId); s_logger.debug("Creating monitoring services on " + router + " start..."); // get the list of sevices for this network to monitor final List services = new ArrayList(); if (_networkModel.isProviderSupportServiceInNetwork(network.getId(), Service.Dhcp, Provider.VirtualRouter) || _networkModel.isProviderSupportServiceInNetwork(network.getId(), Service.Dns, Provider.VirtualRouter)) { final MonitoringServiceVO dhcpService = _monitorServiceDao.getServiceByName(MonitoringService.Service.Dhcp.toString()); if (dhcpService != null) { services.add(dhcpService); } } if (_networkModel.isProviderSupportServiceInNetwork(network.getId(), Service.Lb, Provider.VirtualRouter)) { final MonitoringServiceVO lbService = _monitorServiceDao.getServiceByName(MonitoringService.Service.LoadBalancing.toString()); if (lbService != null) { services.add(lbService); } } final List defaultServices = _monitorServiceDao.listDefaultServices(true); services.addAll(defaultServices); final List servicesTO = new ArrayList(); for (final MonitoringServiceVO service : services) { final MonitorServiceTO serviceTO = new MonitorServiceTO(service.getService(), service.getProcessName(), service.getServiceName(), service.getServicePath(), service.getServicePidFile(), service.isDefaultService()); servicesTO.add(serviceTO); } // TODO : This is a hacking fix // at VR startup time, information in VirtualMachineProfile may not updated to DB yet, // getRouterControlIp() may give wrong IP under basic network mode in VMware environment final NicProfile controlNic = getControlNic(profile); final SetMonitorServiceCommand command = new SetMonitorServiceCommand(servicesTO); command.setAccessDetail(NetworkElementCommand.ROUTER_IP, controlNic.getIp4Address()); command.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(networkId, router.getId())); command.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); if (!add) { command.setAccessDetail(NetworkElementCommand.ROUTER_MONITORING_ENABLE, add.toString()); } cmds.addCommand("monitor", command); } protected NicProfile getControlNic(final VirtualMachineProfile profile) { final DomainRouterVO router = _routerDao.findById(profile.getId()); final DataCenterVO dcVo = _dcDao.findById(router.getDataCenterId()); NicProfile controlNic = null; if (profile.getHypervisorType() == HypervisorType.VMware && dcVo.getNetworkType() == NetworkType.Basic) { // TODO this is a ugly to test hypervisor type here // for basic network mode, we will use the guest NIC for control NIC for (final NicProfile nic : profile.getNics()) { if (nic.getTrafficType() == TrafficType.Guest && nic.getIp4Address() != null) { controlNic = nic; } } } else { for (final NicProfile nic : profile.getNics()) { if (nic.getTrafficType() == TrafficType.Control && nic.getIp4Address() != null) { controlNic = nic; } } } return controlNic; } protected void finalizeSshAndVersionAndNetworkUsageOnStart(final Commands cmds, final VirtualMachineProfile profile, final DomainRouterVO router, final NicProfile controlNic) { final DomainRouterVO vr = _routerDao.findById(profile.getId()); cmds.addCommand("checkSsh", new CheckSshCommand(profile.getInstanceName(), controlNic.getIp4Address(), 3922)); // Update router template/scripts version final GetDomRVersionCmd command = new GetDomRVersionCmd(); command.setAccessDetail(NetworkElementCommand.ROUTER_IP, controlNic.getIp4Address()); command.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); cmds.addCommand("getDomRVersion", command); // Network usage command to create iptables rules final boolean forVpc = vr.getVpcId() != null; if (!forVpc) cmds.addCommand("networkUsage", new NetworkUsageCommand(controlNic.getIp4Address(), router.getHostName(), "create", forVpc)); } protected void finalizeUserDataAndDhcpOnStart(final Commands cmds, final DomainRouterVO router, final Provider provider, final Long guestNetworkId) { if (_networkModel.isProviderSupportServiceInNetwork(guestNetworkId, Service.Dhcp, provider)) { // Resend dhcp s_logger.debug("Reapplying dhcp entries as a part of domR " + router + " start..."); createDhcpEntryCommandsForVMs(router, cmds, guestNetworkId); } if (_networkModel.isProviderSupportServiceInNetwork(guestNetworkId, Service.UserData, provider)) { // Resend user data s_logger.debug("Reapplying vm data (userData and metaData) entries as a part of domR " + router + " start..."); createVmDataCommandForVMs(router, cmds, guestNetworkId); } } protected void finalizeNetworkRulesForNetwork(final Commands cmds, final DomainRouterVO router, final Provider provider, final Long guestNetworkId) { s_logger.debug("Resending ipAssoc, port forwarding, load balancing rules as a part of Virtual router start"); final ArrayList publicIps = getPublicIpsToApply(router, provider, guestNetworkId); final List firewallRulesEgress = new ArrayList(); // Fetch firewall Egress rules. if (_networkModel.isProviderSupportServiceInNetwork(guestNetworkId, Service.Firewall, provider)) { firewallRulesEgress.addAll(_rulesDao.listByNetworkPurposeTrafficType(guestNetworkId, Purpose.Firewall, FirewallRule.TrafficType.Egress)); } // Re-apply firewall Egress rules s_logger.debug("Found " + firewallRulesEgress.size() + " firewall Egress rule(s) to apply as a part of domR " + router + " start."); if (!firewallRulesEgress.isEmpty()) { createFirewallRulesCommands(firewallRulesEgress, router, cmds, guestNetworkId); } if (publicIps != null && !publicIps.isEmpty()) { final List vpns = new ArrayList(); final List pfRules = new ArrayList(); final List staticNatFirewallRules = new ArrayList(); final List staticNats = new ArrayList(); final List firewallRulesIngress = new ArrayList(); //Get information about all the rules (StaticNats and StaticNatRules; PFVPN to reapply on domR start) for (final PublicIpAddress ip : publicIps) { if (_networkModel.isProviderSupportServiceInNetwork(guestNetworkId, Service.PortForwarding, provider)) { pfRules.addAll(_pfRulesDao.listForApplication(ip.getId())); } if (_networkModel.isProviderSupportServiceInNetwork(guestNetworkId, Service.StaticNat, provider)) { staticNatFirewallRules.addAll(_rulesDao.listByIpAndPurpose(ip.getId(), Purpose.StaticNat)); } if (_networkModel.isProviderSupportServiceInNetwork(guestNetworkId, Service.Firewall, provider)) { firewallRulesIngress.addAll(_rulesDao.listByIpAndPurpose(ip.getId(), Purpose.Firewall)); } if (_networkModel.isProviderSupportServiceInNetwork(guestNetworkId, Service.Vpn, provider)) { final RemoteAccessVpn vpn = _vpnDao.findByPublicIpAddress(ip.getId()); if (vpn != null) { vpns.add(vpn); } } if (_networkModel.isProviderSupportServiceInNetwork(guestNetworkId, Service.StaticNat, provider)) { if (ip.isOneToOneNat()) { final StaticNatImpl staticNat = new StaticNatImpl(ip.getAccountId(), ip.getDomainId(), guestNetworkId, ip.getId(), ip.getVmIp(), false); staticNats.add(staticNat); } } } // Re-apply static nats s_logger.debug("Found " + staticNats.size() + " static nat(s) to apply as a part of domR " + router + " start."); if (!staticNats.isEmpty()) { createApplyStaticNatCommands(staticNats, router, cmds, guestNetworkId); } // Re-apply firewall Ingress rules s_logger.debug("Found " + firewallRulesIngress.size() + " firewall Ingress rule(s) to apply as a part of domR " + router + " start."); if (!firewallRulesIngress.isEmpty()) { createFirewallRulesCommands(firewallRulesIngress, router, cmds, guestNetworkId); } // Re-apply port forwarding rules s_logger.debug("Found " + pfRules.size() + " port forwarding rule(s) to apply as a part of domR " + router + " start."); if (!pfRules.isEmpty()) { createApplyPortForwardingRulesCommands(pfRules, router, cmds, guestNetworkId); } // Re-apply static nat rules s_logger.debug("Found " + staticNatFirewallRules.size() + " static nat rule(s) to apply as a part of domR " + router + " start."); if (!staticNatFirewallRules.isEmpty()) { final List staticNatRules = new ArrayList(); for (final FirewallRule rule : staticNatFirewallRules) { staticNatRules.add(_rulesMgr.buildStaticNatRule(rule, false)); } createApplyStaticNatRulesCommands(staticNatRules, router, cmds, guestNetworkId); } // Re-apply vpn rules s_logger.debug("Found " + vpns.size() + " vpn(s) to apply as a part of domR " + router + " start."); if (!vpns.isEmpty()) { for (final RemoteAccessVpn vpn : vpns) { createApplyVpnCommands(true, vpn, router, cmds); } } final List lbs = _loadBalancerDao.listByNetworkIdAndScheme(guestNetworkId, Scheme.Public); final List lbRules = new ArrayList(); if (_networkModel.isProviderSupportServiceInNetwork(guestNetworkId, Service.Lb, provider)) { // Re-apply load balancing rules for (final LoadBalancerVO lb : lbs) { final List dstList = _lbMgr.getExistingDestinations(lb.getId()); final List policyList = _lbMgr.getStickinessPolicies(lb.getId()); final List hcPolicyList = _lbMgr.getHealthCheckPolicies(lb.getId()); final Ip sourceIp = _networkModel.getPublicIpAddress(lb.getSourceIpAddressId()).getAddress(); final LbSslCert sslCert = _lbMgr.getLbSslCert(lb.getId()); final LoadBalancingRule loadBalancing = new LoadBalancingRule(lb, dstList, policyList, hcPolicyList, sourceIp, sslCert, lb.getLbProtocol()); lbRules.add(loadBalancing); } } s_logger.debug("Found " + lbRules.size() + " load balancing rule(s) to apply as a part of domR " + router + " start."); if (!lbRules.isEmpty()) { createApplyLoadBalancingRulesCommands(lbRules, router, cmds, guestNetworkId); } } //Reapply dhcp and dns configuration. final Network guestNetwork = _networkDao.findById(guestNetworkId); if (guestNetwork.getGuestType() == GuestType.Shared && _networkModel.isProviderSupportServiceInNetwork(guestNetworkId, Service.Dhcp, provider)) { final Map dhcpCapabilities = _networkSvc.getNetworkOfferingServiceCapabilities(_networkOfferingDao.findById(_networkDao.findById(guestNetworkId).getNetworkOfferingId()), Service.Dhcp); final String supportsMultipleSubnets = dhcpCapabilities.get(Network.Capability.DhcpAccrossMultipleSubnets); if (supportsMultipleSubnets != null && Boolean.valueOf(supportsMultipleSubnets)) { final List revokedIpAliasVOs = _nicIpAliasDao.listByNetworkIdAndState(guestNetworkId, NicIpAlias.state.revoked); s_logger.debug("Found" + revokedIpAliasVOs.size() + "ip Aliases to revoke on the router as a part of dhcp configuration"); removeRevokedIpAliasFromDb(revokedIpAliasVOs); final List aliasVOs = _nicIpAliasDao.listByNetworkIdAndState(guestNetworkId, NicIpAlias.state.active); s_logger.debug("Found" + aliasVOs.size() + "ip Aliases to apply on the router as a part of dhcp configuration"); final List activeIpAliasTOs = new ArrayList(); for (final NicIpAliasVO aliasVO : aliasVOs) { activeIpAliasTOs.add(new IpAliasTO(aliasVO.getIp4Address(), aliasVO.getNetmask(), aliasVO.getAliasCount().toString())); } if (activeIpAliasTOs.size() != 0) { createIpAlias(router, activeIpAliasTOs, guestNetworkId, cmds); configDnsMasq(router, _networkDao.findById(guestNetworkId), cmds); } } } } private void removeRevokedIpAliasFromDb(final List revokedIpAliasVOs) { for (final NicIpAliasVO ipalias : revokedIpAliasVOs) { _nicIpAliasDao.expunge(ipalias.getId()); } } protected void finalizeIpAssocForNetwork(final Commands cmds, final VirtualRouter router, final Provider provider, final Long guestNetworkId, final Map vlanMacAddress) { final ArrayList publicIps = getPublicIpsToApply(router, provider, guestNetworkId); if (publicIps != null && !publicIps.isEmpty()) { s_logger.debug("Found " + publicIps.size() + " ip(s) to apply as a part of domR " + router + " start."); // Re-apply public ip addresses - should come before PF/LB/VPN if (_networkModel.isProviderSupportServiceInNetwork(guestNetworkId, Service.Firewall, provider)) { createAssociateIPCommands(router, publicIps, cmds, 0); } } } protected ArrayList getPublicIpsToApply(final VirtualRouter router, final Provider provider, final Long guestNetworkId, final com.cloud.network.IpAddress.State... skipInStates) { final long ownerId = router.getAccountId(); final List userIps; final Network guestNetwork = _networkDao.findById(guestNetworkId); if (guestNetwork.getGuestType() == GuestType.Shared) { // ignore the account id for the shared network userIps = _networkModel.listPublicIpsAssignedToGuestNtwk(guestNetworkId, null); } else { userIps = _networkModel.listPublicIpsAssignedToGuestNtwk(ownerId, guestNetworkId, null); } final List allPublicIps = new ArrayList(); if (userIps != null && !userIps.isEmpty()) { boolean addIp = true; for (final IpAddress userIp : userIps) { if (skipInStates != null) { for (final IpAddress.State stateToSkip : skipInStates) { if (userIp.getState() == stateToSkip) { s_logger.debug("Skipping ip address " + userIp + " in state " + userIp.getState()); addIp = false; break; } } } if (addIp) { final IPAddressVO ipVO = _ipAddressDao.findById(userIp.getId()); final PublicIp publicIp = PublicIp.createFromAddrAndVlan(ipVO, _vlanDao.findById(userIp.getVlanId())); allPublicIps.add(publicIp); } } } //Get public Ips that should be handled by router final Network network = _networkDao.findById(guestNetworkId); final Map> ipToServices = _networkModel.getIpToServices(allPublicIps, false, true); final Map> providerToIpList = _networkModel.getProviderToIpList(network, ipToServices); // Only cover virtual router for now, if ELB use it this need to be modified final ArrayList publicIps = providerToIpList.get(provider); return publicIps; } @Override public boolean finalizeStart(final VirtualMachineProfile profile, final long hostId, final Commands cmds, final ReservationContext context) { DomainRouterVO router = _routerDao.findById(profile.getId()); //process all the answers for (Answer answer : cmds.getAnswers()) { // handle any command failures if (!answer.getResult()) { String cmdClassName = answer.getClass().getCanonicalName().replace("Answer", "Command"); String errorMessage = "Command: " + cmdClassName + " failed while starting virtual router"; String errorDetails = "Details: " + answer.getDetails() + " " + answer.toString(); //add alerts for the failed commands _alertMgr.sendAlert(AlertService.AlertType.ALERT_TYPE_DOMAIN_ROUTER, router.getDataCenterId(), router.getPodIdToDeployIn(), errorMessage, errorDetails); s_logger.warn(errorMessage); //Stop the router if any of the commands failed return false; } } // at this point, all the router command are successful. boolean result = true; //Get guest networks info final List guestNetworks = new ArrayList(); final List routerNics = _nicDao.listByVmId(profile.getId()); for (final Nic nic : routerNics) { final Network network = _networkModel.getNetwork(nic.getNetworkId()); if (network.getTrafficType() == TrafficType.Guest) { guestNetworks.add(network); if (nic.getBroadcastUri().getScheme().equals("pvlan")) { final NicProfile nicProfile = new NicProfile(nic, network, nic.getBroadcastUri(), nic.getIsolationUri(), 0, false, "pvlan-nic"); result = setupDhcpForPvlan(true, router, router.getHostId(), nicProfile); } } } if (result) { GetDomRVersionAnswer versionAnswer = (GetDomRVersionAnswer)cmds.getAnswer("getDomRVersion"); router.setTemplateVersion(versionAnswer.getTemplateVersion()); router.setScriptsVersion(versionAnswer.getScriptsVersion()); router = _routerDao.persist(router, guestNetworks); } return result; } @Override public void finalizeStop(final VirtualMachineProfile profile, final Answer answer) { if (answer != null) { final VirtualMachine vm = profile.getVirtualMachine(); final DomainRouterVO domR = _routerDao.findById(vm.getId()); processStopOrRebootAnswer(domR, answer); final List routerNics = _nicDao.listByVmId(profile.getId()); for (final Nic nic : routerNics) { final Network network = _networkModel.getNetwork(nic.getNetworkId()); if (network.getTrafficType() == TrafficType.Guest && nic.getBroadcastUri() != null && nic.getBroadcastUri().getScheme().equals("pvlan")) { final NicProfile nicProfile = new NicProfile(nic, network, nic.getBroadcastUri(), nic.getIsolationUri(), 0, false, "pvlan-nic"); setupDhcpForPvlan(false, domR, domR.getHostId(), nicProfile); } } } } @Override public void finalizeExpunge(final VirtualMachine vm) { } @Override public boolean startRemoteAccessVpn(final Network network, final RemoteAccessVpn vpn, final List routers) throws ResourceUnavailableException { if (routers == null || routers.isEmpty()) { s_logger.warn("Failed to start remote access VPN: no router found for account and zone"); throw new ResourceUnavailableException("Failed to start remote access VPN: no router found for account and zone", DataCenter.class, network.getDataCenterId()); } for (final VirtualRouter router : routers) { if (router.getState() != State.Running) { s_logger.warn("Failed to start remote access VPN: router not in right state " + router.getState()); throw new ResourceUnavailableException("Failed to start remote access VPN: router not in right state " + router.getState(), DataCenter.class, network.getDataCenterId()); } final Commands cmds = new Commands(Command.OnError.Stop); createApplyVpnCommands(true, vpn, router, cmds); if (!sendCommandsToRouter(router, cmds)) { throw new AgentUnavailableException("Unable to send commands to virtual router ", router.getHostId()); } Answer answer = cmds.getAnswer("users"); if (!answer.getResult()) { s_logger.error("Unable to start vpn: unable add users to vpn in zone " + router.getDataCenterId() + " for account " + vpn.getAccountId() + " on domR: " + router.getInstanceName() + " due to " + answer.getDetails()); throw new ResourceUnavailableException("Unable to start vpn: Unable to add users to vpn in zone " + router.getDataCenterId() + " for account " + vpn.getAccountId() + " on domR: " + router.getInstanceName() + " due to " + answer.getDetails(), DataCenter.class, router.getDataCenterId()); } answer = cmds.getAnswer("startVpn"); if (!answer.getResult()) { s_logger.error("Unable to start vpn in zone " + router.getDataCenterId() + " for account " + vpn.getAccountId() + " on domR: " + router.getInstanceName() + " due to " + answer.getDetails()); throw new ResourceUnavailableException("Unable to start vpn in zone " + router.getDataCenterId() + " for account " + vpn.getAccountId() + " on domR: " + router.getInstanceName() + " due to " + answer.getDetails(), DataCenter.class, router.getDataCenterId()); } } return true; } @Override public boolean deleteRemoteAccessVpn(final Network network, final RemoteAccessVpn vpn, final List routers) throws ResourceUnavailableException { if (routers == null || routers.isEmpty()) { s_logger.warn("Failed to delete remote access VPN: no router found for account and zone"); throw new ResourceUnavailableException("Failed to delete remote access VPN", DataCenter.class, network.getDataCenterId()); } boolean result = true; for (final VirtualRouter router : routers) { if (router.getState() == State.Running) { final Commands cmds = new Commands(Command.OnError.Continue); createApplyVpnCommands(false, vpn, router, cmds); result = result && sendCommandsToRouter(router, cmds); } else if (router.getState() == State.Stopped) { s_logger.debug("Router " + router + " is in Stopped state, not sending deleteRemoteAccessVpn command to it"); continue; } else { s_logger.warn("Failed to delete remote access VPN: domR " + router + " is not in right state " + router.getState()); throw new ResourceUnavailableException("Failed to delete remote access VPN: domR is not in right state " + router.getState(), DataCenter.class, network.getDataCenterId()); } } return result; } private DomainRouterVO start(DomainRouterVO router, final User user, final Account caller, final Map params, final DeploymentPlan planToDeploy) throws StorageUnavailableException, InsufficientCapacityException, ConcurrentOperationException, ResourceUnavailableException { s_logger.debug("Starting router " + router); try { _itMgr.advanceStart(router.getUuid(), params, planToDeploy, null); } catch (final OperationTimedoutException e) { throw new ResourceUnavailableException("Starting router " + router + " failed! " + e.toString(), DataCenter.class, router.getDataCenterId()); } if (router.isStopPending()) { s_logger.info("Clear the stop pending flag of router " + router.getHostName() + " after start router successfully!"); router.setStopPending(false); router = _routerDao.persist(router); } // We don't want the failure of VPN Connection affect the status of router, so we try to make connection // only after router start successfully final Long vpcId = router.getVpcId(); if (vpcId != null) { _s2sVpnMgr.reconnectDisconnectedVpnByVpc(vpcId); } return _routerDao.findById(router.getId()); } @Override public DomainRouterVO stop(final VirtualRouter router, final boolean forced, final User user, final Account caller) throws ConcurrentOperationException, ResourceUnavailableException { s_logger.debug("Stopping router " + router); try { _itMgr.advanceStop(router.getUuid(), forced); return _routerDao.findById(router.getId()); } catch (final OperationTimedoutException e) { throw new CloudRuntimeException("Unable to stop " + router, e); } } @Override public boolean configDhcpForSubnet(final Network network, final NicProfile nic, final VirtualMachineProfile profile, final DeployDestination dest, final List routers) throws ResourceUnavailableException { final UserVmVO vm = _userVmDao.findById(profile.getId()); _userVmDao.loadDetails(vm); //Asuming we have only one router per network For Now. final DomainRouterVO router = routers.get(0); if (router.getState() != State.Running) { s_logger.warn("Failed to configure dhcp: router not in running state"); throw new ResourceUnavailableException("Unable to assign ip addresses, domR is not in right state " + router.getState(), DataCenter.class, network.getDataCenterId()); } //check if this is not the primary subnet. final NicVO domr_guest_nic = _nicDao.findByInstanceIdAndIpAddressAndVmtype(router.getId(), _nicDao.getIpAddress(nic.getNetworkId(), router.getId()), VirtualMachine.Type.DomainRouter); //check if the router ip address and the vm ip address belong to same subnet. //if they do not belong to same netwoek check for the alias ips. if not create one. // This should happen only in case of Basic and Advanced SG enabled networks. if (!NetUtils.sameSubnet(domr_guest_nic.getIp4Address(), nic.getIp4Address(), nic.getNetmask())) { final List aliasIps = _nicIpAliasDao.listByNetworkIdAndState(domr_guest_nic.getNetworkId(), NicIpAlias.state.active); boolean ipInVmsubnet = false; for (final NicIpAliasVO alias : aliasIps) { //check if any of the alias ips belongs to the Vm's subnet. if (NetUtils.sameSubnet(alias.getIp4Address(), nic.getIp4Address(), nic.getNetmask())) { ipInVmsubnet = true; break; } } PublicIp routerPublicIP = null; String routerAliasIp = null; final DataCenter dc = _dcDao.findById(router.getDataCenterId()); if (ipInVmsubnet == false) { try { if (network.getTrafficType() == TrafficType.Guest && network.getGuestType() == GuestType.Shared) { _podDao.findById(vm.getPodIdToDeployIn()); final Account caller = CallContext.current().getCallingAccount(); final List vlanList = _vlanDao.listVlansByNetworkIdAndGateway(network.getId(), nic.getGateway()); final List vlanDbIdList = new ArrayList(); for (final VlanVO vlan : vlanList) { vlanDbIdList.add(vlan.getId()); } if (dc.getNetworkType() == NetworkType.Basic) { routerPublicIP = _ipAddrMgr.assignPublicIpAddressFromVlans(router.getDataCenterId(), vm.getPodIdToDeployIn(), caller, Vlan.VlanType.DirectAttached, vlanDbIdList, nic.getNetworkId(), null, false); } else { routerPublicIP = _ipAddrMgr.assignPublicIpAddressFromVlans(router.getDataCenterId(), null, caller, Vlan.VlanType.DirectAttached, vlanDbIdList, nic.getNetworkId(), null, false); } routerAliasIp = routerPublicIP.getAddress().addr(); } } catch (final InsufficientAddressCapacityException e) { s_logger.info(e.getMessage()); s_logger.info("unable to configure dhcp for this VM."); return false; } //this means we did not create a ip alis on the router. final NicIpAliasVO alias = new NicIpAliasVO(domr_guest_nic.getId(), routerAliasIp, router.getId(), CallContext.current().getCallingAccountId(), network.getDomainId(), nic.getNetworkId(), nic.getGateway(), nic.getNetmask()); alias.setAliasCount((routerPublicIP.getIpMacAddress())); _nicIpAliasDao.persist(alias); final List ipaliasTo = new ArrayList(); ipaliasTo.add(new IpAliasTO(routerAliasIp, alias.getNetmask(), alias.getAliasCount().toString())); final Commands cmds = new Commands(Command.OnError.Stop); createIpAlias(router, ipaliasTo, alias.getNetworkId(), cmds); //also add the required configuration to the dnsmasq for supporting dhcp and dns on the new ip. configDnsMasq(router, network, cmds); final boolean result = sendCommandsToRouter(router, cmds); if (result == false) { final NicIpAliasVO ipAliasVO = _nicIpAliasDao.findByInstanceIdAndNetworkId(network.getId(), router.getId()); final PublicIp routerPublicIPFinal = routerPublicIP; Transaction.execute(new TransactionCallbackNoReturn() { @Override public void doInTransactionWithoutResult(final TransactionStatus status) { _nicIpAliasDao.expunge(ipAliasVO.getId()); _ipAddressDao.unassignIpAddress(routerPublicIPFinal.getId()); } }); throw new CloudRuntimeException("failed to configure ip alias on the router as a part of dhcp config"); } } return true; } return true; } @Override public boolean removeDhcpSupportForSubnet(final Network network, final List routers) throws ResourceUnavailableException { if (routers == null || routers.isEmpty()) { s_logger.warn("Failed to add/remove VPN users: no router found for account and zone"); throw new ResourceUnavailableException("Unable to assign ip addresses, domR doesn't exist for network " + network.getId(), DataCenter.class, network.getDataCenterId()); } for (final DomainRouterVO router : routers) { if (router.getState() != State.Running) { s_logger.warn("Failed to add/remove VPN users: router not in running state"); throw new ResourceUnavailableException("Unable to assign ip addresses, domR is not in right state " + router.getState(), DataCenter.class, network.getDataCenterId()); } final Commands cmds = new Commands(Command.OnError.Continue); final List revokedIpAliasVOs = _nicIpAliasDao.listByNetworkIdAndState(network.getId(), NicIpAlias.state.revoked); s_logger.debug("Found" + revokedIpAliasVOs.size() + "ip Aliases to revoke on the router as a part of dhcp configuration"); final List revokedIpAliasTOs = new ArrayList(); for (final NicIpAliasVO revokedAliasVO : revokedIpAliasVOs) { revokedIpAliasTOs.add(new IpAliasTO(revokedAliasVO.getIp4Address(), revokedAliasVO.getNetmask(), revokedAliasVO.getAliasCount().toString())); } final List aliasVOs = _nicIpAliasDao.listByNetworkIdAndState(network.getId(), NicIpAlias.state.active); s_logger.debug("Found" + aliasVOs.size() + "ip Aliases to apply on the router as a part of dhcp configuration"); final List activeIpAliasTOs = new ArrayList(); for (final NicIpAliasVO aliasVO : aliasVOs) { activeIpAliasTOs.add(new IpAliasTO(aliasVO.getIp4Address(), aliasVO.getNetmask(), aliasVO.getAliasCount().toString())); } createDeleteIpAliasCommand(router, revokedIpAliasTOs, activeIpAliasTOs, network.getId(), cmds); configDnsMasq(router, network, cmds); final boolean result = sendCommandsToRouter(router, cmds); if (result) { Transaction.execute(new TransactionCallbackNoReturn() { @Override public void doInTransactionWithoutResult(final TransactionStatus status) { for (final NicIpAliasVO revokedAliasVO : revokedIpAliasVOs) { _nicIpAliasDao.expunge(revokedAliasVO.getId()); } } }); return true; } } return false; } @Override public boolean applyDhcpEntry(final Network network, final NicProfile nic, final VirtualMachineProfile profile, final DeployDestination dest, final List routers) throws ResourceUnavailableException { if (s_logger.isTraceEnabled()) { s_logger.trace("applyDhcpEntry(" + network.getCidr() + ", " + nic.getMacAddress() + ", " + profile.getUuid() + ", " + dest.getHost() + ", " + routers + ")"); } final UserVmVO vm = _userVmDao.findById(profile.getId()); _userVmDao.loadDetails(vm); final VirtualMachineProfile updatedProfile = profile; final boolean isZoneBasic = (dest.getDataCenter().getNetworkType() == NetworkType.Basic); final Long podId = isZoneBasic ? dest.getPod().getId() : null; boolean podLevelException = false; //for user vm in Basic zone we should try to re-deploy vm in a diff pod if it fails to deploy in original pod; so throwing exception with Pod scope if (isZoneBasic && podId != null && updatedProfile.getVirtualMachine().getType() == VirtualMachine.Type.User && network.getTrafficType() == TrafficType.Guest && network.getGuestType() == Network.GuestType.Shared) { podLevelException = true; } return applyRules(network, routers, "dhcp entry", podLevelException, podId, true, new RuleApplier() { @Override public boolean execute(final Network network, final VirtualRouter router) throws ResourceUnavailableException { //for basic zone, send dhcp/dns information to all routers in the basic network only when _dnsBasicZoneUpdates is set to "all" value final Commands cmds = new Commands(Command.OnError.Stop); if (!(isZoneBasic && router.getPodIdToDeployIn().longValue() != podId.longValue() && _dnsBasicZoneUpdates.equalsIgnoreCase("pod"))) { final NicVO nicVo = _nicDao.findById(nic.getId()); createDhcpEntryCommand(router, vm, nicVo, cmds); return sendCommandsToRouter(router, cmds); } return true; } }); } private void createDeleteIpAliasCommand(final DomainRouterVO router, final List deleteIpAliasTOs, final List createIpAliasTos, final long networkId, final Commands cmds) { final String routerip = getRouterIpInNetwork(networkId, router.getId()); final DataCenterVO dcVo = _dcDao.findById(router.getDataCenterId()); final DeleteIpAliasCommand deleteIpaliasCmd = new DeleteIpAliasCommand(routerip, deleteIpAliasTOs, createIpAliasTos); deleteIpaliasCmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); deleteIpaliasCmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); deleteIpaliasCmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, routerip); deleteIpaliasCmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); cmds.addCommand("deleteIpalias", deleteIpaliasCmd); } private NicVO findDefaultDnsIp(final long userVmId) { final NicVO defaultNic = _nicDao.findDefaultNicForVM(userVmId); //check if DNS provider is the domR if (!_networkModel.isProviderSupportServiceInNetwork(defaultNic.getNetworkId(), Service.Dns, Provider.VirtualRouter)) { return null; } final NetworkOffering offering = _networkOfferingDao.findById(_networkDao.findById(defaultNic.getNetworkId()).getNetworkOfferingId()); if (offering.getRedundantRouter()) { return findGatewayIp(userVmId); } final DataCenter dc = _dcDao.findById(_networkModel.getNetwork(defaultNic.getNetworkId()).getDataCenterId()); final boolean isZoneBasic = (dc.getNetworkType() == NetworkType.Basic); //find domR's nic in the network NicVO domrDefaultNic; if (isZoneBasic) { domrDefaultNic = _nicDao.findByNetworkIdTypeAndGateway(defaultNic.getNetworkId(), VirtualMachine.Type.DomainRouter, defaultNic.getGateway()); } else { domrDefaultNic = _nicDao.findByNetworkIdAndType(defaultNic.getNetworkId(), VirtualMachine.Type.DomainRouter); } return domrDefaultNic; } private NicVO findGatewayIp(final long userVmId) { final NicVO defaultNic = _nicDao.findDefaultNicForVM(userVmId); return defaultNic; } @Override public boolean applyUserData(final Network network, final NicProfile nic, final VirtualMachineProfile profile, final DeployDestination dest, final List routers) throws ResourceUnavailableException { final UserVmVO vm = _userVmDao.findById(profile.getId()); _userVmDao.loadDetails(vm); final VirtualMachineProfile updatedProfile = profile; final boolean isZoneBasic = (dest.getDataCenter().getNetworkType() == NetworkType.Basic); final Long podId = isZoneBasic ? dest.getPod().getId() : null; boolean podLevelException = false; //for user vm in Basic zone we should try to re-deploy vm in a diff pod if it fails to deploy in original pod; so throwing exception with Pod scope if (isZoneBasic && podId != null && updatedProfile.getVirtualMachine().getType() == VirtualMachine.Type.User && network.getTrafficType() == TrafficType.Guest && network.getGuestType() == Network.GuestType.Shared) { podLevelException = true; } return applyRules(network, routers, "userdata and password entry", podLevelException, podId, false, new RuleApplier() { @Override public boolean execute(final Network network, final VirtualRouter router) throws ResourceUnavailableException { //for basic zone, send vm data/password information only to the router in the same pod final Commands cmds = new Commands(Command.OnError.Stop); if (!(isZoneBasic && router.getPodIdToDeployIn().longValue() != podId.longValue())) { final NicVO nicVo = _nicDao.findById(nic.getId()); createPasswordCommand(router, updatedProfile, nicVo, cmds); createVmDataCommand(router, vm, nicVo, vm.getDetail("SSH.PublicKey"), cmds); return sendCommandsToRouter(router, cmds); } return true; } }); } protected void createApplyVpnUsersCommand(final List users, final VirtualRouter router, final Commands cmds) { final List addUsers = new ArrayList(); final List removeUsers = new ArrayList(); for (final VpnUser user : users) { if (user.getState() == VpnUser.State.Add || user.getState() == VpnUser.State.Active) { addUsers.add(user); } else if (user.getState() == VpnUser.State.Revoke) { removeUsers.add(user); } } final VpnUsersCfgCommand cmd = new VpnUsersCfgCommand(addUsers, removeUsers); cmd.setAccessDetail(NetworkElementCommand.ACCOUNT_ID, String.valueOf(router.getAccountId())); cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); final DataCenterVO dcVo = _dcDao.findById(router.getDataCenterId()); cmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); cmds.addCommand("users", cmd); } @Override //FIXME add partial success and STOP state support public String[] applyVpnUsers(final Network network, final List users, final List routers) throws ResourceUnavailableException { if (routers == null || routers.isEmpty()) { s_logger.warn("Failed to add/remove VPN users: no router found for account and zone"); throw new ResourceUnavailableException("Unable to assign ip addresses, domR doesn't exist for network " + network.getId(), DataCenter.class, network.getDataCenterId()); } boolean agentResults = true; for (final DomainRouterVO router : routers) { if (router.getState() != State.Running) { s_logger.warn("Failed to add/remove VPN users: router not in running state"); throw new ResourceUnavailableException("Unable to assign ip addresses, domR is not in right state " + router.getState(), DataCenter.class, network.getDataCenterId()); } final Commands cmds = new Commands(Command.OnError.Continue); createApplyVpnUsersCommand(users, router, cmds); // Currently we receive just one answer from the agent. In the future we have to parse individual answers and set // results accordingly final boolean agentResult = sendCommandsToRouter(router, cmds); agentResults = agentResults && agentResult; } final String[] result = new String[users.size()]; for (int i = 0; i < result.length; i++) { if (agentResults) { result[i] = null; } else { result[i] = String.valueOf(agentResults); } } return result; } @Override @ActionEvent(eventType = EventTypes.EVENT_ROUTER_START, eventDescription = "starting router Vm", async = true) public VirtualRouter startRouter(final long id) throws ResourceUnavailableException, InsufficientCapacityException, ConcurrentOperationException { return startRouter(id, true); } @Override public VirtualRouter startRouter(final long routerId, final boolean reprogramNetwork) throws ResourceUnavailableException, InsufficientCapacityException, ConcurrentOperationException { final Account caller = CallContext.current().getCallingAccount(); final User callerUser = _accountMgr.getActiveUser(CallContext.current().getCallingUserId()); // verify parameters DomainRouterVO router = _routerDao.findById(routerId); if (router == null) { throw new InvalidParameterValueException("Unable to find router by id " + routerId + "."); } _accountMgr.checkAccess(caller, null, true, router); final Account owner = _accountMgr.getAccount(router.getAccountId()); // Check if all networks are implemented for the domR; if not - implement them final DataCenter dc = _dcDao.findById(router.getDataCenterId()); HostPodVO pod = null; if (router.getPodIdToDeployIn() != null) { pod = _podDao.findById(router.getPodIdToDeployIn()); } final DeployDestination dest = new DeployDestination(dc, pod, null, null); final ReservationContext context = new ReservationContextImpl(null, null, callerUser, owner); final List nics = _nicDao.listByVmId(routerId); for (final NicVO nic : nics) { if (!_networkMgr.startNetwork(nic.getNetworkId(), dest, context)) { s_logger.warn("Failed to start network id=" + nic.getNetworkId() + " as a part of domR start"); throw new CloudRuntimeException("Failed to start network id=" + nic.getNetworkId() + " as a part of domR start"); } } //After start network, check if it's already running router = _routerDao.findById(routerId); if (router.getState() == State.Running) { return router; } final UserVO user = _userDao.findById(CallContext.current().getCallingUserId()); final Map params = new HashMap(); if (reprogramNetwork) { params.put(Param.ReProgramGuestNetworks, true); } else { params.put(Param.ReProgramGuestNetworks, false); } final VirtualRouter virtualRouter = startVirtualRouter(router, user, caller, params); if (virtualRouter == null) { throw new CloudRuntimeException("Failed to start router with id " + routerId); } return virtualRouter; } private void createAssociateIPCommands(final VirtualRouter router, final List ips, final Commands cmds, final long vmId) { // Ensure that in multiple vlans case we first send all ip addresses of vlan1, then all ip addresses of vlan2, etc.. final Map> vlanIpMap = new HashMap>(); for (final PublicIpAddress ipAddress : ips) { final String vlanTag = ipAddress.getVlanTag(); ArrayList ipList = vlanIpMap.get(vlanTag); if (ipList == null) { ipList = new ArrayList(); } //domR doesn't support release for sourceNat IP address; so reset the state if (ipAddress.isSourceNat() && ipAddress.getState() == IpAddress.State.Releasing) { ipAddress.setState(IpAddress.State.Allocated); } ipList.add(ipAddress); vlanIpMap.put(vlanTag, ipList); } final List nics = _nicDao.listByVmId(router.getId()); String baseMac = null; for (final NicVO nic : nics) { final NetworkVO nw = _networkDao.findById(nic.getNetworkId()); if (nw.getTrafficType() == TrafficType.Public) { baseMac = nic.getMacAddress(); break; } } for (final Map.Entry> vlanAndIp : vlanIpMap.entrySet()) { final List ipAddrList = vlanAndIp.getValue(); // Source nat ip address should always be sent first Collections.sort(ipAddrList, new Comparator() { @Override public int compare(final PublicIpAddress o1, final PublicIpAddress o2) { final boolean s1 = o1.isSourceNat(); final boolean s2 = o2.isSourceNat(); return (s1 ^ s2) ? ((s1 ^ true) ? 1 : -1) : 0; } }); // Get network rate - required for IpAssoc final Integer networkRate = _networkModel.getNetworkRate(ipAddrList.get(0).getNetworkId(), router.getId()); final Network network = _networkModel.getNetwork(ipAddrList.get(0).getNetworkId()); final IpAddressTO[] ipsToSend = new IpAddressTO[ipAddrList.size()]; int i = 0; boolean firstIP = true; for (final PublicIpAddress ipAddr : ipAddrList) { final boolean add = (ipAddr.getState() == IpAddress.State.Releasing ? false : true); boolean sourceNat = ipAddr.isSourceNat(); /* enable sourceNAT for the first ip of the public interface */ if (firstIP) { sourceNat = true; } final String vlanId = ipAddr.getVlanTag(); final String vlanGateway = ipAddr.getGateway(); final String vlanNetmask = ipAddr.getNetmask(); String vifMacAddress = null; // For non-source nat IP, set the mac to be something based on first public nic's MAC // We cannot depends on first ip because we need to deal with first ip of other nics if (!ipAddr.isSourceNat() && ipAddr.getVlanId() != 0) { vifMacAddress = NetUtils.generateMacOnIncrease(baseMac, ipAddr.getVlanId()); } else { vifMacAddress = ipAddr.getMacAddress(); } final IpAddressTO ip = new IpAddressTO(ipAddr.getAccountId(), ipAddr.getAddress().addr(), add, firstIP, sourceNat, vlanId, vlanGateway, vlanNetmask, vifMacAddress, networkRate, ipAddr.isOneToOneNat()); ip.setTrafficType(network.getTrafficType()); ip.setNetworkName(_networkModel.getNetworkTag(router.getHypervisorType(), network)); ipsToSend[i++] = ip; /* send the firstIP = true for the first Add, this is to create primary on interface*/ if (!firstIP || add) { firstIP = false; } } final IpAssocCommand cmd = new IpAssocCommand(ipsToSend); cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); cmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(ipAddrList.get(0).getAssociatedWithNetworkId(), router.getId())); cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); final DataCenterVO dcVo = _dcDao.findById(router.getDataCenterId()); cmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); cmds.addCommand("IPAssocCommand", cmd); } } private void createApplyPortForwardingRulesCommands(final List rules, final VirtualRouter router, final Commands cmds, final long guestNetworkId) { List rulesTO = null; if (rules != null) { rulesTO = new ArrayList(); for (final PortForwardingRule rule : rules) { final IpAddress sourceIp = _networkModel.getIp(rule.getSourceIpAddressId()); final PortForwardingRuleTO ruleTO = new PortForwardingRuleTO(rule, null, sourceIp.getAddress().addr()); rulesTO.add(ruleTO); } } SetPortForwardingRulesCommand cmd = null; if (router.getVpcId() != null) { cmd = new SetPortForwardingRulesVpcCommand(rulesTO); } else { cmd = new SetPortForwardingRulesCommand(rulesTO); } cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); cmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(guestNetworkId, router.getId())); cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); final DataCenterVO dcVo = _dcDao.findById(router.getDataCenterId()); cmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); cmds.addCommand(cmd); } private void createApplyStaticNatRulesCommands(final List rules, final VirtualRouter router, final Commands cmds, final long guestNetworkId) { List rulesTO = null; if (rules != null) { rulesTO = new ArrayList(); for (final StaticNatRule rule : rules) { final IpAddress sourceIp = _networkModel.getIp(rule.getSourceIpAddressId()); final StaticNatRuleTO ruleTO = new StaticNatRuleTO(rule, null, sourceIp.getAddress().addr(), rule.getDestIpAddress()); rulesTO.add(ruleTO); } } final SetStaticNatRulesCommand cmd = new SetStaticNatRulesCommand(rulesTO, router.getVpcId()); cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); cmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(guestNetworkId, router.getId())); cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); final DataCenterVO dcVo = _dcDao.findById(router.getDataCenterId()); cmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); cmds.addCommand(cmd); } private void createApplyLoadBalancingRulesCommands(final List rules, final VirtualRouter router, final Commands cmds, final long guestNetworkId) { final LoadBalancerTO[] lbs = new LoadBalancerTO[rules.size()]; int i = 0; // We don't support VR to be inline currently final boolean inline = false; for (final LoadBalancingRule rule : rules) { final boolean revoked = (rule.getState().equals(FirewallRule.State.Revoke)); final String protocol = rule.getProtocol(); final String algorithm = rule.getAlgorithm(); final String uuid = rule.getUuid(); final String srcIp = rule.getSourceIp().addr(); final int srcPort = rule.getSourcePortStart(); final List destinations = rule.getDestinations(); final List stickinessPolicies = rule.getStickinessPolicies(); final LoadBalancerTO lb = new LoadBalancerTO(uuid, srcIp, srcPort, protocol, algorithm, revoked, false, inline, destinations, stickinessPolicies); lbs[i++] = lb; } String routerPublicIp = null; if (router instanceof DomainRouterVO) { final DomainRouterVO domr = _routerDao.findById(router.getId()); routerPublicIp = domr.getPublicIpAddress(); } final Network guestNetwork = _networkModel.getNetwork(guestNetworkId); final Nic nic = _nicDao.findByNtwkIdAndInstanceId(guestNetwork.getId(), router.getId()); final NicProfile nicProfile = new NicProfile(nic, guestNetwork, nic.getBroadcastUri(), nic.getIsolationUri(), _networkModel.getNetworkRate(guestNetwork.getId(), router.getId()), _networkModel.isSecurityGroupSupportedInNetwork(guestNetwork), _networkModel.getNetworkTag(router.getHypervisorType(), guestNetwork)); final NetworkOffering offering = _networkOfferingDao.findById(guestNetwork.getNetworkOfferingId()); String maxconn = null; if (offering.getConcurrentConnections() == null) { maxconn = _configDao.getValue(Config.NetworkLBHaproxyMaxConn.key()); } else { maxconn = offering.getConcurrentConnections().toString(); } final LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(lbs, routerPublicIp, getRouterIpInNetwork(guestNetworkId, router.getId()), router.getPrivateIpAddress(), _itMgr.toNicTO( nicProfile, router.getHypervisorType()), router.getVpcId(), maxconn, offering.isKeepAliveEnabled()); cmd.lbStatsVisibility = _configDao.getValue(Config.NetworkLBHaproxyStatsVisbility.key()); cmd.lbStatsUri = _configDao.getValue(Config.NetworkLBHaproxyStatsUri.key()); cmd.lbStatsAuth = _configDao.getValue(Config.NetworkLBHaproxyStatsAuth.key()); cmd.lbStatsPort = _configDao.getValue(Config.NetworkLBHaproxyStatsPort.key()); cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); cmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(guestNetworkId, router.getId())); cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); final DataCenterVO dcVo = _dcDao.findById(router.getDataCenterId()); cmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); cmds.addCommand(cmd); } protected String getVpnCidr(final RemoteAccessVpn vpn) { final Network network = _networkDao.findById(vpn.getNetworkId()); return network.getCidr(); } protected void createApplyVpnCommands(final boolean isCreate, final RemoteAccessVpn vpn, final VirtualRouter router, final Commands cmds) { final List vpnUsers = _vpnUsersDao.listByAccount(vpn.getAccountId()); createApplyVpnUsersCommand(vpnUsers, router, cmds); final IpAddress ip = _networkModel.getIp(vpn.getServerAddressId()); final String cidr = getVpnCidr(vpn); final RemoteAccessVpnCfgCommand startVpnCmd = new RemoteAccessVpnCfgCommand(isCreate, ip.getAddress().addr(), vpn.getLocalIp(), vpn.getIpRange(), vpn.getIpsecPresharedKey(), (vpn.getVpcId() != null)); startVpnCmd.setLocalCidr(cidr); startVpnCmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); startVpnCmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); final DataCenterVO dcVo = _dcDao.findById(router.getDataCenterId()); startVpnCmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); cmds.addCommand("startVpn", startVpnCmd); } private void createPasswordCommand(final VirtualRouter router, final VirtualMachineProfile profile, final NicVO nic, final Commands cmds) { final String password = (String)profile.getParameter(VirtualMachineProfile.Param.VmPassword); final DataCenterVO dcVo = _dcDao.findById(router.getDataCenterId()); // password should be set only on default network element if (password != null && nic.isDefaultNic()) { final String encodedPassword = PasswordGenerator.rot13(password); final SavePasswordCommand cmd = new SavePasswordCommand(encodedPassword, nic.getIp4Address(), profile.getVirtualMachine().getHostName(), _networkModel.getExecuteInSeqNtwkElmtCmd()); cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); cmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(nic.getNetworkId(), router.getId())); cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); cmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); cmds.addCommand("password", cmd); } } private void createVmDataCommand(final VirtualRouter router, final UserVm vm, final NicVO nic, final String publicKey, final Commands cmds) { final String serviceOffering = _serviceOfferingDao.findByIdIncludingRemoved(vm.getId(), vm.getServiceOfferingId()).getDisplayText(); final String zoneName = _dcDao.findById(router.getDataCenterId()).getName(); cmds.addCommand( "vmdata", generateVmDataCommand(router, nic.getIp4Address(), vm.getUserData(), serviceOffering, zoneName, nic.getIp4Address(), vm.getHostName(), vm.getInstanceName(), vm.getId(), vm.getUuid(), publicKey, nic.getNetworkId())); } private void createVmDataCommandForVMs(final DomainRouterVO router, final Commands cmds, final long guestNetworkId) { final List vms = _userVmDao.listByNetworkIdAndStates(guestNetworkId, State.Running, State.Migrating, State.Stopping); final DataCenterVO dc = _dcDao.findById(router.getDataCenterId()); for (final UserVmVO vm : vms) { boolean createVmData = true; if (dc.getNetworkType() == NetworkType.Basic && router.getPodIdToDeployIn().longValue() != vm.getPodIdToDeployIn().longValue()) { createVmData = false; } if (createVmData) { final NicVO nic = _nicDao.findByNtwkIdAndInstanceId(guestNetworkId, vm.getId()); if (nic != null) { s_logger.debug("Creating user data entry for vm " + vm + " on domR " + router); createVmDataCommand(router, vm, nic, null, cmds); } } } } private void createDhcpEntryCommand(final VirtualRouter router, final UserVm vm, final NicVO nic, final Commands cmds) { final DhcpEntryCommand dhcpCommand = new DhcpEntryCommand(nic.getMacAddress(), nic.getIp4Address(), vm.getHostName(), nic.getIp6Address(), _networkModel.getExecuteInSeqNtwkElmtCmd()); final DataCenterVO dcVo = _dcDao.findById(router.getDataCenterId()); final Nic defaultNic = findGatewayIp(vm.getId()); String gatewayIp = defaultNic.getGateway(); if (gatewayIp != null && !gatewayIp.equals(nic.getGateway())) { gatewayIp = "0.0.0.0"; } dhcpCommand.setDefaultRouter(gatewayIp); dhcpCommand.setIp6Gateway(nic.getIp6Gateway()); String ipaddress = null; final NicVO domrDefaultNic = findDefaultDnsIp(vm.getId()); if (domrDefaultNic != null) { ipaddress = domrDefaultNic.getIp4Address(); } dhcpCommand.setDefaultDns(ipaddress); dhcpCommand.setDuid(NetUtils.getDuidLL(nic.getMacAddress())); dhcpCommand.setDefault(nic.isDefaultNic()); dhcpCommand.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); dhcpCommand.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); dhcpCommand.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(nic.getNetworkId(), router.getId())); dhcpCommand.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); cmds.addCommand("dhcp", dhcpCommand); } private void configDnsMasq(final VirtualRouter router, final Network network, final Commands cmds) { final DataCenterVO dcVo = _dcDao.findById(router.getDataCenterId()); final List ipAliasVOList = _nicIpAliasDao.listByNetworkIdAndState(network.getId(), NicIpAlias.state.active); final List ipList = new ArrayList(); final NicVO router_guest_nic = _nicDao.findByNtwkIdAndInstanceId(network.getId(), router.getId()); final String cidr = NetUtils.getCidrFromGatewayAndNetmask(router_guest_nic.getGateway(), router_guest_nic.getNetmask()); final String[] cidrPair = cidr.split("\\/"); final String cidrAddress = cidrPair[0]; final long cidrSize = Long.parseLong(cidrPair[1]); final String startIpOfSubnet = NetUtils.getIpRangeStartIpFromCidr(cidrAddress, cidrSize); ipList.add(new DhcpTO(router_guest_nic.getIp4Address(), router_guest_nic.getGateway(), router_guest_nic.getNetmask(), startIpOfSubnet)); for (final NicIpAliasVO ipAliasVO : ipAliasVOList) { final DhcpTO DhcpTO = new DhcpTO(ipAliasVO.getIp4Address(), ipAliasVO.getGateway(), ipAliasVO.getNetmask(), ipAliasVO.getStartIpOfSubnet()); if (s_logger.isTraceEnabled()) { s_logger.trace("configDnsMasq : adding ip {" + DhcpTO.getGateway() + ", " + DhcpTO.getNetmask() + ", " + DhcpTO.getRouterIp() + ", " + DhcpTO.getStartIpOfSubnet() + "}"); } ipList.add(DhcpTO); ipAliasVO.setVmId(router.getId()); } _dcDao.findById(router.getDataCenterId()); final DnsMasqConfigCommand dnsMasqConfigCmd = new DnsMasqConfigCommand(ipList); dnsMasqConfigCmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); dnsMasqConfigCmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); dnsMasqConfigCmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(network.getId(), router.getId())); dnsMasqConfigCmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); cmds.addCommand("dnsMasqConfig", dnsMasqConfigCmd); } private void createIpAlias(final VirtualRouter router, final List ipAliasTOs, final Long networkid, final Commands cmds) { final String routerip = getRouterIpInNetwork(networkid, router.getId()); final DataCenterVO dcVo = _dcDao.findById(router.getDataCenterId()); final CreateIpAliasCommand ipaliasCmd = new CreateIpAliasCommand(routerip, ipAliasTOs); ipaliasCmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); ipaliasCmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); ipaliasCmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, routerip); ipaliasCmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); cmds.addCommand("ipalias", ipaliasCmd); } private void createDhcpEntryCommandsForVMs(final DomainRouterVO router, final Commands cmds, final long guestNetworkId) { final List vms = _userVmDao.listByNetworkIdAndStates(guestNetworkId, State.Running, State.Migrating, State.Stopping); final DataCenterVO dc = _dcDao.findById(router.getDataCenterId()); for (final UserVmVO vm : vms) { boolean createDhcp = true; if (dc.getNetworkType() == NetworkType.Basic && router.getPodIdToDeployIn().longValue() != vm.getPodIdToDeployIn().longValue() && _dnsBasicZoneUpdates.equalsIgnoreCase("pod")) { createDhcp = false; } if (createDhcp) { final NicVO nic = _nicDao.findByNtwkIdAndInstanceId(guestNetworkId, vm.getId()); if (nic != null) { s_logger.debug("Creating dhcp entry for vm " + vm + " on domR " + router + "."); createDhcpEntryCommand(router, vm, nic, cmds); } } } } protected boolean sendCommandsToRouter(final VirtualRouter router, final Commands cmds) throws AgentUnavailableException { if(!checkRouterVersion(router)){ s_logger.debug("Router requires upgrade. Unable to send command to router:" + router.getId() + ", router template version : " + router.getTemplateVersion() + ", minimal required version : " + MinVRVersion); throw new CloudRuntimeException("Unable to send command. Upgrade in progress. Please contact administrator."); } Answer[] answers = null; try { answers = _agentMgr.send(router.getHostId(), cmds); } catch (final OperationTimedoutException e) { s_logger.warn("Timed Out", e); throw new AgentUnavailableException("Unable to send commands to virtual router ", router.getHostId(), e); } if (answers == null) { return false; } if (answers.length != cmds.size()) { return false; } // FIXME: Have to return state for individual command in the future boolean result = true; if (answers.length > 0) { for (final Answer answer : answers) { if (!answer.getResult()) { result = false; break; } } } return result; } protected void handleSingleWorkingRedundantRouter(final List connectedRouters, final List disconnectedRouters, final String reason) throws ResourceUnavailableException { if (connectedRouters.isEmpty() || disconnectedRouters.isEmpty()) { return; } if (connectedRouters.size() != 1 || disconnectedRouters.size() != 1) { s_logger.warn("How many redundant routers do we have?? "); return; } if (!connectedRouters.get(0).getIsRedundantRouter()) { throw new ResourceUnavailableException("Who is calling this with non-redundant router or non-domain router?", DataCenter.class, connectedRouters.get(0) .getDataCenterId()); } if (!disconnectedRouters.get(0).getIsRedundantRouter()) { throw new ResourceUnavailableException("Who is calling this with non-redundant router or non-domain router?", DataCenter.class, disconnectedRouters.get(0) .getDataCenterId()); } final DomainRouterVO connectedRouter = (DomainRouterVO)connectedRouters.get(0); DomainRouterVO disconnectedRouter = (DomainRouterVO)disconnectedRouters.get(0); if (s_logger.isDebugEnabled()) { s_logger.debug("About to stop the router " + disconnectedRouter.getInstanceName() + " due to: " + reason); } final String title = "Virtual router " + disconnectedRouter.getInstanceName() + " would be stopped after connecting back, due to " + reason; final String context = "Virtual router (name: " + disconnectedRouter.getInstanceName() + ", id: " + disconnectedRouter.getId() + ") would be stopped after connecting back, due to: " + reason; _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_DOMAIN_ROUTER, disconnectedRouter.getDataCenterId(), disconnectedRouter.getPodIdToDeployIn(), title, context); disconnectedRouter.setStopPending(true); disconnectedRouter = _routerDao.persist(disconnectedRouter); final int connRouterPR = getRealPriority(connectedRouter); final int disconnRouterPR = getRealPriority(disconnectedRouter); if (connRouterPR < disconnRouterPR) { //connRouterPR < disconnRouterPR, they won't equal at anytime if (!connectedRouter.getIsPriorityBumpUp()) { final BumpUpPriorityCommand command = new BumpUpPriorityCommand(); command.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(connectedRouter.getId())); command.setAccessDetail(NetworkElementCommand.ROUTER_NAME, connectedRouter.getInstanceName()); final Answer answer = _agentMgr.easySend(connectedRouter.getHostId(), command); if (!answer.getResult()) { s_logger.error("Failed to bump up " + connectedRouter.getInstanceName() + "'s priority! " + answer.getDetails()); } } else { final String t = "Can't bump up virtual router " + connectedRouter.getInstanceName() + "'s priority due to it's already bumped up!"; _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_DOMAIN_ROUTER, connectedRouter.getDataCenterId(), connectedRouter.getPodIdToDeployIn(), t, t); } } } @Override public boolean associatePublicIP(final Network network, final List ipAddress, final List routers) throws ResourceUnavailableException { if (ipAddress == null || ipAddress.isEmpty()) { s_logger.debug("No ip association rules to be applied for network " + network.getId()); return true; } return applyRules(network, routers, "ip association", false, null, false, new RuleApplier() { @Override public boolean execute(final Network network, final VirtualRouter router) throws ResourceUnavailableException { final Commands cmds = new Commands(Command.OnError.Continue); createAssociateIPCommands(router, ipAddress, cmds, 0); return sendCommandsToRouter(router, cmds); } }); } @Override public boolean applyFirewallRules(final Network network, final List rules, final List routers) throws ResourceUnavailableException { if (rules == null || rules.isEmpty()) { s_logger.debug("No firewall rules to be applied for network " + network.getId()); return true; } return applyRules(network, routers, "firewall rules", false, null, false, new RuleApplier() { @Override public boolean execute(final Network network, final VirtualRouter router) throws ResourceUnavailableException { if (rules.get(0).getPurpose() == Purpose.LoadBalancing) { // for load balancer we have to resend all lb rules for the network final List lbs = _loadBalancerDao.listByNetworkIdAndScheme(network.getId(), Scheme.Public); final List lbRules = new ArrayList(); for (final LoadBalancerVO lb : lbs) { final List dstList = _lbMgr.getExistingDestinations(lb.getId()); final List policyList = _lbMgr.getStickinessPolicies(lb.getId()); final List hcPolicyList = _lbMgr.getHealthCheckPolicies(lb.getId()); final LbSslCert sslCert = _lbMgr.getLbSslCert(lb.getId()); final Ip sourceIp = _networkModel.getPublicIpAddress(lb.getSourceIpAddressId()).getAddress(); final LoadBalancingRule loadBalancing = new LoadBalancingRule(lb, dstList, policyList, hcPolicyList, sourceIp, sslCert, lb.getLbProtocol()); lbRules.add(loadBalancing); } return sendLBRules(router, lbRules, network.getId()); } else if (rules.get(0).getPurpose() == Purpose.PortForwarding) { return sendPortForwardingRules(router, (List)rules, network.getId()); } else if (rules.get(0).getPurpose() == Purpose.StaticNat) { return sendStaticNatRules(router, (List)rules, network.getId()); } else if (rules.get(0).getPurpose() == Purpose.Firewall) { return sendFirewallRules(router, (List)rules, network.getId()); } else { s_logger.warn("Unable to apply rules of purpose: " + rules.get(0).getPurpose()); return false; } } }); } @Override public boolean applyLoadBalancingRules(final Network network, final List rules, final List routers) throws ResourceUnavailableException { if (rules == null || rules.isEmpty()) { s_logger.debug("No lb rules to be applied for network " + network.getId()); return true; } return applyRules(network, routers, "loadbalancing rules", false, null, false, new RuleApplier() { @Override public boolean execute(final Network network, final VirtualRouter router) throws ResourceUnavailableException { // for load balancer we have to resend all lb rules for the network final List lbs = _loadBalancerDao.listByNetworkIdAndScheme(network.getId(), Scheme.Public); final List lbRules = new ArrayList(); for (final LoadBalancerVO lb : lbs) { final List dstList = _lbMgr.getExistingDestinations(lb.getId()); final List policyList = _lbMgr.getStickinessPolicies(lb.getId()); final List hcPolicyList = _lbMgr.getHealthCheckPolicies(lb.getId()); final LbSslCert sslCert = _lbMgr.getLbSslCert(lb.getId()); final Ip sourceIp = _networkModel.getPublicIpAddress(lb.getSourceIpAddressId()).getAddress(); final LoadBalancingRule loadBalancing = new LoadBalancingRule(lb, dstList, policyList, hcPolicyList, sourceIp, sslCert, lb.getLbProtocol()); lbRules.add(loadBalancing); } return sendLBRules(router, lbRules, network.getId()); } }); } protected boolean sendLBRules(final VirtualRouter router, final List rules, final long guestNetworkId) throws ResourceUnavailableException { final Commands cmds = new Commands(Command.OnError.Continue); createApplyLoadBalancingRulesCommands(rules, router, cmds, guestNetworkId); return sendCommandsToRouter(router, cmds); } protected boolean sendPortForwardingRules(final VirtualRouter router, final List rules, final long guestNetworkId) throws ResourceUnavailableException { final Commands cmds = new Commands(Command.OnError.Continue); createApplyPortForwardingRulesCommands(rules, router, cmds, guestNetworkId); return sendCommandsToRouter(router, cmds); } protected boolean sendStaticNatRules(final VirtualRouter router, final List rules, final long guestNetworkId) throws ResourceUnavailableException { final Commands cmds = new Commands(Command.OnError.Continue); createApplyStaticNatRulesCommands(rules, router, cmds, guestNetworkId); return sendCommandsToRouter(router, cmds); } @Override public List getRoutersForNetwork(final long networkId) { final List routers = _routerDao.findByNetwork(networkId); final List vrs = new ArrayList(routers.size()); for (final DomainRouterVO router : routers) { vrs.add(router); } return vrs; } private void createFirewallRulesCommands(final List rules, final VirtualRouter router, final Commands cmds, final long guestNetworkId) { List rulesTO = null; String systemRule = null; Boolean defaultEgressPolicy = false; if (rules != null) { if (rules.size() > 0) { if (rules.get(0).getTrafficType() == FirewallRule.TrafficType.Egress && rules.get(0).getType() == FirewallRule.FirewallRuleType.System) { systemRule = String.valueOf(FirewallRule.FirewallRuleType.System); } } rulesTO = new ArrayList(); for (final FirewallRule rule : rules) { _rulesDao.loadSourceCidrs((FirewallRuleVO)rule); final FirewallRule.TrafficType traffictype = rule.getTrafficType(); if (traffictype == FirewallRule.TrafficType.Ingress) { final IpAddress sourceIp = _networkModel.getIp(rule.getSourceIpAddressId()); final FirewallRuleTO ruleTO = new FirewallRuleTO(rule, null, sourceIp.getAddress().addr(), Purpose.Firewall, traffictype); rulesTO.add(ruleTO); } else if (rule.getTrafficType() == FirewallRule.TrafficType.Egress) { final NetworkVO network = _networkDao.findById(guestNetworkId); final NetworkOfferingVO offering = _networkOfferingDao.findById(network.getNetworkOfferingId()); defaultEgressPolicy = offering.getEgressDefaultPolicy(); assert (rule.getSourceIpAddressId() == null) : "ipAddressId should be null for egress firewall rule. "; final FirewallRuleTO ruleTO = new FirewallRuleTO(rule, null, "", Purpose.Firewall, traffictype, defaultEgressPolicy); rulesTO.add(ruleTO); } } } final SetFirewallRulesCommand cmd = new SetFirewallRulesCommand(rulesTO); cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); cmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(guestNetworkId, router.getId())); cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); final DataCenterVO dcVo = _dcDao.findById(router.getDataCenterId()); cmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); if (systemRule != null) { cmd.setAccessDetail(NetworkElementCommand.FIREWALL_EGRESS_DEFAULT, systemRule); } else { cmd.setAccessDetail(NetworkElementCommand.FIREWALL_EGRESS_DEFAULT, String.valueOf(defaultEgressPolicy)); } cmds.addCommand(cmd); } protected boolean sendFirewallRules(final VirtualRouter router, final List rules, final long guestNetworkId) throws ResourceUnavailableException { final Commands cmds = new Commands(Command.OnError.Continue); createFirewallRulesCommands(rules, router, cmds, guestNetworkId); return sendCommandsToRouter(router, cmds); } @Override public String getDnsBasicZoneUpdate() { return _dnsBasicZoneUpdates; } protected interface RuleApplier { boolean execute(Network network, VirtualRouter router) throws ResourceUnavailableException; } protected boolean applyRules(final Network network, final List routers, final String typeString, final boolean isPodLevelException, final Long podId, final boolean failWhenDisconnect, final RuleApplier applier) throws ResourceUnavailableException { if (routers == null || routers.isEmpty()) { s_logger.warn("Unable to apply " + typeString + ", virtual router doesn't exist in the network " + network.getId()); throw new ResourceUnavailableException("Unable to apply " + typeString, DataCenter.class, network.getDataCenterId()); } final DataCenter dc = _dcDao.findById(network.getDataCenterId()); final boolean isZoneBasic = (dc.getNetworkType() == NetworkType.Basic); // isPodLevelException and podId is only used for basic zone assert !((!isZoneBasic && isPodLevelException) || (isZoneBasic && isPodLevelException && podId == null)); final List connectedRouters = new ArrayList(); final List disconnectedRouters = new ArrayList(); boolean result = true; final String msg = "Unable to apply " + typeString + " on disconnected router "; for (final VirtualRouter router : routers) { if (router.getState() == State.Running) { s_logger.debug("Applying " + typeString + " in network " + network); if (router.isStopPending()) { if (_hostDao.findById(router.getHostId()).getState() == Status.Up) { throw new ResourceUnavailableException("Unable to process due to the stop pending router " + router.getInstanceName() + " haven't been stopped after it's host coming back!", DataCenter.class, router.getDataCenterId()); } s_logger.debug("Router " + router.getInstanceName() + " is stop pending, so not sending apply " + typeString + " commands to the backend"); continue; } try { result = applier.execute(network, router); connectedRouters.add(router); } catch (final AgentUnavailableException e) { s_logger.warn(msg + router.getInstanceName(), e); disconnectedRouters.add(router); } //If rules fail to apply on one domR and not due to disconnection, no need to proceed with the rest if (!result) { if (isZoneBasic && isPodLevelException) { throw new ResourceUnavailableException("Unable to apply " + typeString + " on router ", Pod.class, podId); } throw new ResourceUnavailableException("Unable to apply " + typeString + " on router ", DataCenter.class, router.getDataCenterId()); } } else if (router.getState() == State.Stopped || router.getState() == State.Stopping) { s_logger.debug("Router " + router.getInstanceName() + " is in " + router.getState() + ", so not sending apply " + typeString + " commands to the backend"); } else { s_logger.warn("Unable to apply " + typeString + ", virtual router is not in the right state " + router.getState()); if (isZoneBasic && isPodLevelException) { throw new ResourceUnavailableException("Unable to apply " + typeString + ", virtual router is not in the right state", Pod.class, podId); } throw new ResourceUnavailableException("Unable to apply " + typeString + ", virtual router is not in the right state", DataCenter.class, router.getDataCenterId()); } } if (!connectedRouters.isEmpty()) { if (!isZoneBasic && !disconnectedRouters.isEmpty() && disconnectedRouters.get(0).getIsRedundantRouter()) { // These disconnected redundant virtual routers are out of sync now, stop them for synchronization handleSingleWorkingRedundantRouter(connectedRouters, disconnectedRouters, msg); } } else if (!disconnectedRouters.isEmpty()) { for (final VirtualRouter router : disconnectedRouters) { if (s_logger.isDebugEnabled()) { s_logger.debug(msg + router.getInstanceName() + "(" + router.getId() + ")"); } } if (isZoneBasic && isPodLevelException) { throw new ResourceUnavailableException(msg, Pod.class, podId); } throw new ResourceUnavailableException(msg, DataCenter.class, disconnectedRouters.get(0).getDataCenterId()); } result = true; if (failWhenDisconnect) { result = !connectedRouters.isEmpty(); } return result; } @Override public boolean applyStaticNats(final Network network, final List rules, final List routers) throws ResourceUnavailableException { if (rules == null || rules.isEmpty()) { s_logger.debug("No static nat rules to be applied for network " + network.getId()); return true; } return applyRules(network, routers, "static nat rules", false, null, false, new RuleApplier() { @Override public boolean execute(final Network network, final VirtualRouter router) throws ResourceUnavailableException { return applyStaticNat(router, rules, network.getId()); } }); } protected boolean applyStaticNat(final VirtualRouter router, final List rules, final long guestNetworkId) throws ResourceUnavailableException { final Commands cmds = new Commands(Command.OnError.Continue); createApplyStaticNatCommands(rules, router, cmds, guestNetworkId); return sendCommandsToRouter(router, cmds); } private void createApplyStaticNatCommands(final List rules, final VirtualRouter router, final Commands cmds, final long guestNetworkId) { List rulesTO = null; if (rules != null) { rulesTO = new ArrayList(); for (final StaticNat rule : rules) { final IpAddress sourceIp = _networkModel.getIp(rule.getSourceIpAddressId()); final StaticNatRuleTO ruleTO = new StaticNatRuleTO(0, sourceIp.getAddress().addr(), null, null, rule.getDestIpAddress(), null, null, null, rule.isForRevoke(), false); rulesTO.add(ruleTO); } } final SetStaticNatRulesCommand cmd = new SetStaticNatRulesCommand(rulesTO, router.getVpcId()); cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); cmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(guestNetworkId, router.getId())); cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); final DataCenterVO dcVo = _dcDao.findById(router.getDataCenterId()); cmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); cmds.addCommand(cmd); } @Override public int getTimeout() { return -1; } @Override public boolean isRecurring() { return false; } @Override public boolean processAnswers(final long agentId, final long seq, final Answer[] answers) { return false; } @Override public boolean processCommands(final long agentId, final long seq, final Command[] commands) { return false; } @Override public void processConnect(final Host host, final StartupCommand cmd, final boolean forRebalance) throws ConnectionException { final List routers = _routerDao.listIsolatedByHostId(host.getId()); for (DomainRouterVO router : routers) { if (router.isStopPending()) { s_logger.info("Stopping router " + router.getInstanceName() + " due to stop pending flag found!"); final State state = router.getState(); if (state != State.Stopped && state != State.Destroyed) { try { stopRouter(router.getId(), false); } catch (final ResourceUnavailableException e) { s_logger.warn("Fail to stop router " + router.getInstanceName(), e); throw new ConnectionException(false, "Fail to stop router " + router.getInstanceName()); } catch (final ConcurrentOperationException e) { s_logger.warn("Fail to stop router " + router.getInstanceName(), e); throw new ConnectionException(false, "Fail to stop router " + router.getInstanceName()); } } router.setStopPending(false); router = _routerDao.persist(router); } } } @Override public AgentControlAnswer processControlCommand(final long agentId, final AgentControlCommand cmd) { return null; } @Override public boolean processDisconnect(final long agentId, final Status state) { return false; } @Override public boolean processTimeout(final long agentId, final long seq) { return false; } protected String getRouterControlIp(final long routerId) { String routerControlIpAddress = null; final List nics = _nicDao.listByVmId(routerId); for (final NicVO n : nics) { final NetworkVO nc = _networkDao.findById(n.getNetworkId()); if (nc != null && nc.getTrafficType() == TrafficType.Control) { routerControlIpAddress = n.getIp4Address(); // router will have only one control ip break; } } if (routerControlIpAddress == null) { s_logger.warn("Unable to find router's control ip in its attached NICs!. routerId: " + routerId); final DomainRouterVO router = _routerDao.findById(routerId); return router.getPrivateIpAddress(); } return routerControlIpAddress; } protected String getRouterIpInNetwork(final long networkId, final long instanceId) { return _nicDao.getIpAddress(networkId, instanceId); } @Override public void prepareStop(final VirtualMachineProfile profile) { //Collect network usage before stopping Vm final DomainRouterVO router = _routerDao.findById(profile.getVirtualMachine().getId()); if (router == null) { return; } final String privateIP = router.getPrivateIpAddress(); if (privateIP != null) { final boolean forVpc = router.getVpcId() != null; final List routerNics = _nicDao.listByVmId(router.getId()); for (final Nic routerNic : routerNics) { final Network network = _networkModel.getNetwork(routerNic.getNetworkId()); //Send network usage command for public nic in VPC VR //Send network usage command for isolated guest nic of non VPC VR if ((forVpc && network.getTrafficType() == TrafficType.Public) || (!forVpc && network.getTrafficType() == TrafficType.Guest && network.getGuestType() == Network.GuestType.Isolated)) { final NetworkUsageCommand usageCmd = new NetworkUsageCommand(privateIP, router.getHostName(), forVpc, routerNic.getIp4Address()); final String routerType = router.getType().toString(); final UserStatisticsVO previousStats = _userStatsDao.findBy(router.getAccountId(), router.getDataCenterId(), network.getId(), (forVpc ? routerNic.getIp4Address() : null), router.getId(), routerType); NetworkUsageAnswer answer = null; try { answer = (NetworkUsageAnswer)_agentMgr.easySend(router.getHostId(), usageCmd); } catch (final Exception e) { s_logger.warn("Error while collecting network stats from router: " + router.getInstanceName() + " from host: " + router.getHostId(), e); continue; } if (answer != null) { if (!answer.getResult()) { s_logger.warn("Error while collecting network stats from router: " + router.getInstanceName() + " from host: " + router.getHostId() + "; details: " + answer.getDetails()); continue; } try { if ((answer.getBytesReceived() == 0) && (answer.getBytesSent() == 0)) { s_logger.debug("Recieved and Sent bytes are both 0. Not updating user_statistics"); continue; } final NetworkUsageAnswer answerFinal = answer; Transaction.execute(new TransactionCallbackNoReturn() { @Override public void doInTransactionWithoutResult(final TransactionStatus status) { final UserStatisticsVO stats = _userStatsDao.lock(router.getAccountId(), router.getDataCenterId(), network.getId(), (forVpc ? routerNic.getIp4Address() : null), router.getId(), routerType); if (stats == null) { s_logger.warn("unable to find stats for account: " + router.getAccountId()); return; } if (previousStats != null && ((previousStats.getCurrentBytesReceived() != stats.getCurrentBytesReceived()) || (previousStats.getCurrentBytesSent() != stats.getCurrentBytesSent()))) { s_logger.debug("Router stats changed from the time NetworkUsageCommand was sent. " + "Ignoring current answer. Router: " + answerFinal.getRouterName() + " Rcvd: " + answerFinal.getBytesReceived() + "Sent: " + answerFinal.getBytesSent()); return; } if (stats.getCurrentBytesReceived() > answerFinal.getBytesReceived()) { if (s_logger.isDebugEnabled()) { s_logger.debug("Received # of bytes that's less than the last one. " + "Assuming something went wrong and persisting it. Router: " + answerFinal.getRouterName() + " Reported: " + answerFinal.getBytesReceived() + " Stored: " + stats.getCurrentBytesReceived()); } stats.setNetBytesReceived(stats.getNetBytesReceived() + stats.getCurrentBytesReceived()); } stats.setCurrentBytesReceived(answerFinal.getBytesReceived()); if (stats.getCurrentBytesSent() > answerFinal.getBytesSent()) { if (s_logger.isDebugEnabled()) { s_logger.debug("Received # of bytes that's less than the last one. " + "Assuming something went wrong and persisting it. Router: " + answerFinal.getRouterName() + " Reported: " + answerFinal.getBytesSent() + " Stored: " + stats.getCurrentBytesSent()); } stats.setNetBytesSent(stats.getNetBytesSent() + stats.getCurrentBytesSent()); } stats.setCurrentBytesSent(answerFinal.getBytesSent()); if (!_dailyOrHourly) { //update agg bytes stats.setAggBytesSent(stats.getNetBytesSent() + stats.getCurrentBytesSent()); stats.setAggBytesReceived(stats.getNetBytesReceived() + stats.getCurrentBytesReceived()); } _userStatsDao.update(stats.getId(), stats); } }); } catch (final Exception e) { s_logger.warn("Unable to update user statistics for account: " + router.getAccountId() + " Rx: " + answer.getBytesReceived() + "; Tx: " + answer.getBytesSent()); } } } } } } @Override public VirtualRouter findRouter(final long routerId) { return _routerDao.findById(routerId); } @Override public List upgradeRouterTemplate(final UpgradeRouterTemplateCmd cmd) { List routers = new ArrayList(); int params = 0; final Long routerId = cmd.getId(); if (routerId != null) { params++; final DomainRouterVO router = _routerDao.findById(routerId); if (router != null) { routers.add(router); } } final Long domainId = cmd.getDomainId(); if(domainId != null){ final String accountName = cmd.getAccount(); //List by account, if account Name is specified along with domainId if(accountName != null){ final Account account = _accountMgr.getActiveAccountByName(accountName, domainId); if(account == null){ throw new InvalidParameterValueException("Account :"+accountName+" does not exist in domain: "+domainId); } routers = _routerDao.listRunningByAccountId(account.getId()); } else { //List by domainId, account name not specified routers = _routerDao.listRunningByDomain(domainId); } params++; } final Long clusterId = cmd.getClusterId(); if (clusterId != null) { params++; routers = _routerDao.listRunningByClusterId(clusterId); } final Long podId = cmd.getPodId(); if (podId != null) { params++; routers = _routerDao.listRunningByPodId(podId); } final Long zoneId = cmd.getZoneId(); if (zoneId != null) { params++; routers = _routerDao.listRunningByDataCenter(zoneId); } if (params > 1) { throw new InvalidParameterValueException("Multiple parameters not supported. Specify only one among routerId/zoneId/podId/clusterId/accountId/domainId"); } if (routers != null) { return rebootRouters(routers); } return null; } //Checks if the router is at the required version // Compares MS version and router version protected boolean checkRouterVersion(final VirtualRouter router) { if(!routerVersionCheckEnabled.value()){ //Router version check is disabled. return true; } if(router.getTemplateVersion() == null){ return false; } final String trimmedVersion = Version.trimRouterVersion(router.getTemplateVersion()); return (Version.compare(trimmedVersion, MinVRVersion) >= 0); } private List rebootRouters(List routers){ List jobIds = new ArrayList(); for(DomainRouterVO router: routers){ if(!checkRouterVersion(router)){ s_logger.debug("Upgrading template for router: "+router.getId()); Map params = new HashMap(); params.put("ctxUserId", "1"); params.put("ctxAccountId", "" + router.getAccountId()); RebootRouterCmd cmd = new RebootRouterCmd(); ComponentContext.inject(cmd); params.put("id", ""+router.getId()); params.put("ctxStartEventId", "1"); AsyncJobVO job = new AsyncJobVO("", User.UID_SYSTEM, router.getAccountId(), RebootRouterCmd.class.getName(), ApiGsonHelper.getBuilder().create().toJson(params), router.getId(), cmd.getInstanceType() != null ? cmd.getInstanceType().toString() : null); job.setDispatcher(_asyncDispatcher.getName()); long jobId = _asyncMgr.submitAsyncJob(job); jobIds.add(jobId); } else { s_logger.debug("Router: " + router.getId() + " is already at the latest version. No upgrade required"); } } return jobIds; } @Override public String getConfigComponentName() { return VirtualNetworkApplianceManagerImpl.class.getSimpleName(); } @Override public ConfigKey[] getConfigKeys() { return new ConfigKey[] {UseExternalDnsServers, routerVersionCheckEnabled, SetServiceMonitor, RouterAlertsCheckInterval}; } @Override public boolean preStateTransitionEvent(State oldState, VirtualMachine.Event event, State newState, VirtualMachine vo, boolean status, Object opaque) { return true; } @Override public boolean postStateTransitionEvent(State oldState, VirtualMachine.Event event, State newState, VirtualMachine vo, boolean status, Object opaque) { if (oldState == State.Stopped && event == VirtualMachine.Event.FollowAgentPowerOnReport && newState == State.Running) { if (vo.getType() == VirtualMachine.Type.DomainRouter) { s_logger.info("Schedule a router reboot task as router " + vo.getId() + " is powered-on out-of-band. we need to reboot to refresh network rules"); _executor.schedule(new RebootTask(vo.getId()), 1000, TimeUnit.MICROSECONDS); } } return true; } protected class RebootTask extends ManagedContextRunnable { long _routerId; public RebootTask(long routerId) { _routerId = routerId; } @Override protected void runInContext() { try { s_logger.info("Reboot router " + _routerId + " to refresh network rules"); rebootRouter(_routerId, true); } catch (Exception e) { s_logger.warn("Error while rebooting the router", e); } } } protected boolean aggregationExecution(AggregationControlCommand.Action action, Network network, List routers) throws AgentUnavailableException { for (DomainRouterVO router : routers) { AggregationControlCommand cmd = new AggregationControlCommand(action, router.getInstanceName(), getRouterControlIp(router.getId()), getRouterIpInNetwork(network.getId(), router.getId())); Commands cmds = new Commands(cmd); if (!sendCommandsToRouter(router, cmds)) { return false; } } return true; } @Override public boolean prepareAggregatedExecution(Network network, List routers) throws AgentUnavailableException { return aggregationExecution(Action.Start, network, routers); } @Override public boolean completeAggregatedExecution(Network network, List routers) throws AgentUnavailableException { return aggregationExecution(Action.Finish, network, routers); } @Override public boolean cleanupAggregatedExecution(Network network, List routers) throws AgentUnavailableException { return aggregationExecution(Action.Cleanup, network, routers); } }