From 211b60ef648a10be93ea7b4b16b1b54004d56af7 Mon Sep 17 00:00:00 2001 From: Deepak Garg Date: Mon, 30 Jul 2012 11:35:19 +0530 Subject: [PATCH] CS-15644 Implemented UUIDs for templateId, serviceOfferingId, zoneId, domainId replacing the ids being passed to Netscaler in the API url Reviewed By: Vijay V. --- .../cloud/agent/api/to/LoadBalancerTO.java | 50 +- .../cloud/network/as/AutoScaleVmGroup.java | 2 + .../cloud/network/lb/LoadBalancingRule.java | 33 +- .../com/cloud/network/rules/LoadBalancer.java | 7 +- .../network/resource/NetscalerResource.java | 4579 +++++------ ...ExternalLoadBalancerDeviceManagerImpl.java | 9 +- .../cloud/network/as/AutoScaleVmGroupVO.java | 5 + .../network/element/NetscalerElement.java | 28 +- .../lb/ElasticLoadBalancerManagerImpl.java | 126 +- .../lb/LoadBalancingRulesManagerImpl.java | 20 +- .../VirtualNetworkApplianceManagerImpl.java | 6936 +++++++++-------- 11 files changed, 5929 insertions(+), 5866 deletions(-) diff --git a/api/src/com/cloud/agent/api/to/LoadBalancerTO.java b/api/src/com/cloud/agent/api/to/LoadBalancerTO.java index 7a52855d404..e1833c48f5f 100644 --- a/api/src/com/cloud/agent/api/to/LoadBalancerTO.java +++ b/api/src/com/cloud/agent/api/to/LoadBalancerTO.java @@ -30,7 +30,7 @@ import com.cloud.network.lb.LoadBalancingRule.LbStickinessPolicy; import com.cloud.utils.Pair; public class LoadBalancerTO implements Serializable { - Long id; + String uuid; String srcIp; int srcPort; String protocol; @@ -42,11 +42,11 @@ public class LoadBalancerTO implements Serializable { private AutoScaleVmGroupTO autoScaleVmGroupTO; final static int MAX_STICKINESS_POLICIES = 1; - public LoadBalancerTO(Long id, String srcIp, int srcPort, String protocol, String algorithm, boolean revoked, boolean alreadyAdded, List destinations) { - if(destinations == null) { // for autoscaleconfig destinations will be null; + public LoadBalancerTO(String uuid, String srcIp, int srcPort, String protocol, String algorithm, boolean revoked, boolean alreadyAdded, List destinations) { + if (destinations == null) { // for autoscaleconfig destinations will be null; destinations = new ArrayList(); } - this.id = id; + this.uuid = uuid; this.srcIp = srcIp; this.srcPort = srcPort; this.protocol = protocol; @@ -61,7 +61,7 @@ public class LoadBalancerTO implements Serializable { } } - public LoadBalancerTO(Long id, String srcIp, int srcPort, String protocol, String algorithm, boolean revoked, boolean alreadyAdded, List arg_destinations, List stickinessPolicies) { + public LoadBalancerTO(String id, String srcIp, int srcPort, String protocol, String algorithm, boolean revoked, boolean alreadyAdded, List arg_destinations, List stickinessPolicies) { this(id, srcIp, srcPort, protocol, algorithm, revoked, alreadyAdded, arg_destinations); this.stickinessPolicies = null; if (stickinessPolicies != null && stickinessPolicies.size() > 0) { @@ -85,8 +85,8 @@ public class LoadBalancerTO implements Serializable { protected LoadBalancerTO() { } - public Long getId() { - return id; + public String getUuid() { + return uuid; } public String getSrcIp() { @@ -133,7 +133,7 @@ public class LoadBalancerTO implements Serializable { return this.autoScaleVmGroupTO != null; } - public static class StickinessPolicyTO implements Serializable{ + public static class StickinessPolicyTO implements Serializable { private final String _methodName; private final List> _paramsList; @@ -151,7 +151,7 @@ public class LoadBalancerTO implements Serializable { } } - public static class DestinationTO implements Serializable{ + public static class DestinationTO implements Serializable { String destIp; int destPort; boolean revoked; @@ -184,7 +184,7 @@ public class LoadBalancerTO implements Serializable { } } - public static class CounterTO implements Serializable{ + public static class CounterTO implements Serializable { private final String name; private final String source; private final String value; @@ -208,7 +208,7 @@ public class LoadBalancerTO implements Serializable { } } - public static class ConditionTO implements Serializable{ + public static class ConditionTO implements Serializable { private final long threshold; private final String relationalOperator; private final CounterTO counter; @@ -233,7 +233,7 @@ public class LoadBalancerTO implements Serializable { } } - public static class AutoScalePolicyTO implements Serializable{ + public static class AutoScalePolicyTO implements Serializable { private final long id; private final int duration; private final int quietTime; @@ -275,11 +275,11 @@ public class LoadBalancerTO implements Serializable { } } - public static class AutoScaleVmProfileTO implements Serializable{ - private final Long zoneId; - private final Long domainId; - private final Long serviceOfferingId; - private final Long templateId; + public static class AutoScaleVmProfileTO implements Serializable { + private final String zoneId; + private final String domainId; + private final String serviceOfferingId; + private final String templateId; private final String otherDeployParams; private final String snmpCommunity; private final Integer snmpPort; @@ -288,7 +288,7 @@ public class LoadBalancerTO implements Serializable { private final String autoScaleUserApiKey; private final String autoScaleUserSecretKey; - public AutoScaleVmProfileTO(Long zoneId, Long domainId, String cloudStackApiUrl, String autoScaleUserApiKey, String autoScaleUserSecretKey, Long serviceOfferingId, Long templateId, + public AutoScaleVmProfileTO(String zoneId, String domainId, String cloudStackApiUrl, String autoScaleUserApiKey, String autoScaleUserSecretKey, String serviceOfferingId, String templateId, String otherDeployParams, String snmpCommunity, Integer snmpPort, Integer destroyVmGraceperiod) { this.zoneId = zoneId; this.domainId = domainId; @@ -303,19 +303,19 @@ public class LoadBalancerTO implements Serializable { this.autoScaleUserSecretKey = autoScaleUserSecretKey; } - public Long getZoneId() { + public String getZoneId() { return zoneId; } - public Long getDomainId() { + public String getDomainId() { return domainId; } - public Long getServiceOfferingId() { + public String getServiceOfferingId() { return serviceOfferingId; } - public Long getTemplateId() { + public String getTemplateId() { return templateId; } @@ -348,7 +348,7 @@ public class LoadBalancerTO implements Serializable { } } - public static class AutoScaleVmGroupTO implements Serializable{ + public static class AutoScaleVmGroupTO implements Serializable { private final int minMembers; private final int maxMembers; private final int memberPort; @@ -419,9 +419,9 @@ public class LoadBalancerTO implements Serializable { LbAutoScaleVmProfile lbAutoScaleVmProfile = lbAutoScaleVmGroup.getProfile(); AutoScaleVmProfile autoScaleVmProfile = lbAutoScaleVmProfile.getProfile(); - AutoScaleVmProfileTO autoScaleVmProfileTO = new AutoScaleVmProfileTO(autoScaleVmProfile.getZoneId(), autoScaleVmProfile.getDomainId(), + AutoScaleVmProfileTO autoScaleVmProfileTO = new AutoScaleVmProfileTO(lbAutoScaleVmProfile.getZoneId(), lbAutoScaleVmProfile.getDomainId(), lbAutoScaleVmProfile.getCsUrl(), lbAutoScaleVmProfile.getAutoScaleUserApiKey(), lbAutoScaleVmProfile.getAutoScaleUserSecretKey(), - autoScaleVmProfile.getServiceOfferingId(), autoScaleVmProfile.getTemplateId(), autoScaleVmProfile.getOtherDeployParams(), + lbAutoScaleVmProfile.getServiceOfferingId(), lbAutoScaleVmProfile.getTemplateId(), autoScaleVmProfile.getOtherDeployParams(), autoScaleVmProfile.getSnmpCommunity(), autoScaleVmProfile.getSnmpPort(), autoScaleVmProfile.getDestroyVmGraceperiod()); AutoScaleVmGroup autoScaleVmGroup = lbAutoScaleVmGroup.getVmGroup(); diff --git a/api/src/com/cloud/network/as/AutoScaleVmGroup.java b/api/src/com/cloud/network/as/AutoScaleVmGroup.java index 02b8f503890..9d6dc75a99d 100644 --- a/api/src/com/cloud/network/as/AutoScaleVmGroup.java +++ b/api/src/com/cloud/network/as/AutoScaleVmGroup.java @@ -49,4 +49,6 @@ public interface AutoScaleVmGroup extends ControlledEntity { String getState(); + String getUuid(); + } \ No newline at end of file diff --git a/api/src/com/cloud/network/lb/LoadBalancingRule.java b/api/src/com/cloud/network/lb/LoadBalancingRule.java index 976db867b94..61ebfc0c03b 100644 --- a/api/src/com/cloud/network/lb/LoadBalancingRule.java +++ b/api/src/com/cloud/network/lb/LoadBalancingRule.java @@ -75,6 +75,11 @@ public class LoadBalancingRule implements FirewallRule, LoadBalancer { return lb.getAlgorithm(); } + @Override + public String getUuid() { + return lb.getUuid(); + } + @Override public String getXid() { return lb.getXid(); @@ -298,12 +303,20 @@ public class LoadBalancingRule implements FirewallRule, LoadBalancer { private final String autoScaleUserApiKey; private final String autoScaleUserSecretKey; private final String csUrl; + private final String zoneId; + private final String domainId; + private final String serviceOfferingId; + private final String templateId; - public LbAutoScaleVmProfile(AutoScaleVmProfile profile, String autoScaleUserApiKey, String autoScaleUserSecretKey, String csUrl) { + public LbAutoScaleVmProfile(AutoScaleVmProfile profile, String autoScaleUserApiKey, String autoScaleUserSecretKey, String csUrl, String zoneId, String domainId, String serviceOfferingId, String templateId) { this.profile = profile; this.autoScaleUserApiKey = autoScaleUserApiKey; this.autoScaleUserSecretKey = autoScaleUserSecretKey; this.csUrl = csUrl; + this.zoneId = zoneId; + this.domainId = domainId; + this.serviceOfferingId = serviceOfferingId; + this.templateId = templateId; } public AutoScaleVmProfile getProfile() { @@ -317,9 +330,26 @@ public class LoadBalancingRule implements FirewallRule, LoadBalancer { public String getAutoScaleUserSecretKey() { return autoScaleUserSecretKey; } + public String getCsUrl() { return csUrl; } + + public String getZoneId() { + return zoneId; + } + + public String getDomainId() { + return domainId; + } + + public String getServiceOfferingId() { + return serviceOfferingId; + } + + public String getTemplateId() { + return templateId; + } } public static class LbAutoScaleVmGroup { @@ -345,4 +375,5 @@ public class LoadBalancingRule implements FirewallRule, LoadBalancer { return profile; } } + } diff --git a/api/src/com/cloud/network/rules/LoadBalancer.java b/api/src/com/cloud/network/rules/LoadBalancer.java index 122223e70b9..877edb63f15 100644 --- a/api/src/com/cloud/network/rules/LoadBalancer.java +++ b/api/src/com/cloud/network/rules/LoadBalancer.java @@ -13,7 +13,7 @@ package com.cloud.network.rules; /** - * Definition for a LoadBalancer + * Definition for a LoadBalancer */ public interface LoadBalancer extends FirewallRule { @@ -22,9 +22,10 @@ public interface LoadBalancer extends FirewallRule { String getDescription(); int getDefaultPortStart(); - + int getDefaultPortEnd(); String getAlgorithm(); - + + String getUuid(); } diff --git a/core/src/com/cloud/network/resource/NetscalerResource.java b/core/src/com/cloud/network/resource/NetscalerResource.java index f8ae5247349..a32e2952e04 100644 --- a/core/src/com/cloud/network/resource/NetscalerResource.java +++ b/core/src/com/cloud/network/resource/NetscalerResource.java @@ -1,2275 +1,2304 @@ -// Copyright 2012 Citrix Systems, Inc. Licensed under the -// Apache License, Version 2.0 (the "License"); you may not use this -// file except in compliance with the License. Citrix Systems, Inc. -// reserves all rights not expressly granted by the License. -// You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Automatically generated by addcopyright.py at 04/03/2012 -package com.cloud.network.resource; - -import java.util.Formatter; -import java.util.HashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; - -import javax.naming.ConfigurationException; - -import org.apache.log4j.Logger; - -import com.citrix.netscaler.nitro.exception.nitro_exception; -import com.citrix.netscaler.nitro.resource.base.base_response; -import com.citrix.netscaler.nitro.resource.config.basic.server_service_binding; -import com.citrix.netscaler.nitro.resource.config.basic.servicegroup; -import com.citrix.netscaler.nitro.resource.config.basic.servicegroup_lbmonitor_binding; -import com.citrix.netscaler.nitro.resource.config.lb.lbmetrictable; -import com.citrix.netscaler.nitro.resource.config.lb.lbmetrictable_metric_binding; -import com.citrix.netscaler.nitro.resource.config.lb.lbmonitor; -import com.citrix.netscaler.nitro.resource.config.lb.lbmonitor_metric_binding; -import com.citrix.netscaler.nitro.resource.config.lb.lbvserver; -import com.citrix.netscaler.nitro.resource.config.lb.lbvserver_service_binding; -import com.citrix.netscaler.nitro.resource.config.lb.lbvserver_servicegroup_binding; -import com.citrix.netscaler.nitro.resource.config.network.Interface; -import com.citrix.netscaler.nitro.resource.config.network.inat; -import com.citrix.netscaler.nitro.resource.config.network.vlan; -import com.citrix.netscaler.nitro.resource.config.network.vlan_interface_binding; -import com.citrix.netscaler.nitro.resource.config.network.vlan_nsip_binding; -import com.citrix.netscaler.nitro.resource.config.ns.nsconfig; -import com.citrix.netscaler.nitro.resource.config.ns.nshardware; -import com.citrix.netscaler.nitro.resource.config.ns.nsip; -import com.citrix.netscaler.nitro.resource.config.ns.nstimer; -import com.citrix.netscaler.nitro.resource.config.ns.nstimer_autoscalepolicy_binding; -import com.citrix.netscaler.nitro.resource.config.autoscale.*; -import com.citrix.netscaler.nitro.resource.stat.lb.lbvserver_stats; -import com.citrix.netscaler.nitro.service.nitro_service; -import com.citrix.netscaler.nitro.util.filtervalue; -import com.citrix.sdx.nitro.resource.config.device_profile; -import com.citrix.sdx.nitro.resource.config.mps; -import com.citrix.sdx.nitro.resource.config.ns; -import com.citrix.sdx.nitro.resource.config.xen_vpx_image; -import com.cloud.agent.IAgentControl; -import com.cloud.agent.api.Answer; -import com.cloud.agent.api.Command; -import com.cloud.agent.api.ExternalNetworkResourceUsageAnswer; -import com.cloud.agent.api.ExternalNetworkResourceUsageCommand; -import com.cloud.agent.api.MaintainAnswer; -import com.cloud.agent.api.MaintainCommand; -import com.cloud.agent.api.PingCommand; -import com.cloud.agent.api.ReadyAnswer; -import com.cloud.agent.api.ReadyCommand; -import com.cloud.agent.api.StartupCommand; -import com.cloud.agent.api.StartupExternalLoadBalancerCommand; -import com.cloud.agent.api.routing.CreateLoadBalancerApplianceCommand; -import com.cloud.agent.api.routing.DestroyLoadBalancerApplianceCommand; -import com.cloud.agent.api.routing.IpAssocAnswer; -import com.cloud.agent.api.routing.IpAssocCommand; -import com.cloud.agent.api.routing.LoadBalancerConfigCommand; -import com.cloud.agent.api.routing.SetStaticNatRulesAnswer; -import com.cloud.agent.api.routing.SetStaticNatRulesCommand; -import com.cloud.agent.api.to.IpAddressTO; -import com.cloud.agent.api.to.LoadBalancerTO; -import com.cloud.agent.api.to.LoadBalancerTO.AutoScalePolicyTO; -import com.cloud.agent.api.to.LoadBalancerTO.AutoScaleVmGroupTO; -import com.cloud.agent.api.to.LoadBalancerTO.AutoScaleVmProfileTO; -import com.cloud.agent.api.to.LoadBalancerTO.ConditionTO; -import com.cloud.agent.api.to.LoadBalancerTO.CounterTO; -import com.cloud.agent.api.to.LoadBalancerTO.DestinationTO; -import com.cloud.agent.api.to.LoadBalancerTO.StickinessPolicyTO; -import com.cloud.agent.api.to.StaticNatRuleTO; -import com.cloud.api.ApiConstants; -import com.cloud.host.Host; -import com.cloud.host.Host.Type; -import com.cloud.network.rules.LbStickinessMethod.StickinessMethodType; -import com.cloud.resource.ServerResource; -import com.cloud.serializer.GsonHelper; -import com.cloud.utils.NumbersUtil; -import com.cloud.utils.Pair; -import com.cloud.utils.exception.ExecutionException; -import com.cloud.utils.net.NetUtils; -import com.google.gson.Gson; - -class NitroError { - static final int NS_RESOURCE_EXISTS = 273; - static final int NS_RESOURCE_NOT_EXISTS=258; - static final int NS_NO_SERIVCE = 344; - static final int NS_OPERATION_NOT_PERMITTED = 257; - static final int NS_INTERFACE_ALREADY_BOUND_TO_VLAN = 2080; -} - -public class NetscalerResource implements ServerResource { - - - // deployment configuration - private String _name; - private String _zoneId; - private String _physicalNetworkId; - public String _ip; - public String _username; - public String _password; - private String _publicInterface; - private String _privateInterface; - private Integer _numRetries; - private String _guid; - private boolean _inline; - private boolean _isSdx; - private boolean _cloudManaged; - private String _deviceName; - private String _publicIP; - private String _publicIPNetmask; - private String _publicIPGateway; - private String _publicIPVlan; - - private static final Logger s_logger = Logger.getLogger(NetscalerResource.class); - protected Gson _gson; - private String _objectNamePathSep = "-"; - - // interface to interact with VPX and MPX devices - com.citrix.netscaler.nitro.service.nitro_service _netscalerService ; - - // interface to interact with service VM of the SDX appliance - com.citrix.sdx.nitro.service.nitro_service _netscalerSdxService; - - Long _timeout = new Long(100000); - base_response apiCallResult; - - public NetscalerResource () { - _gson = GsonHelper.getGsonLogger(); - } - - @Override - public boolean configure(String name, Map params) throws ConfigurationException { - try { - _name = (String) params.get("name"); - if (_name == null) { - throw new ConfigurationException("Unable to find name in the configuration parameters"); - } - - _zoneId = (String) params.get("zoneId"); - if (_zoneId == null) { - throw new ConfigurationException("Unable to find zone Id in the configuration parameters"); - } - - _physicalNetworkId = (String) params.get("physicalNetworkId"); - if (_physicalNetworkId == null) { - throw new ConfigurationException("Unable to find physical network id in the configuration parameters"); - } - - _ip = (String) params.get("ip"); - if (_ip == null) { - throw new ConfigurationException("Unable to find IP address in the configuration parameters"); - } - - _username = (String) params.get("username"); - if (_username == null) { - throw new ConfigurationException("Unable to find username in the configuration parameters"); - } - - _password = (String) params.get("password"); - if (_password == null) { - throw new ConfigurationException("Unable to find password in the configuration parameters"); - } - - _publicInterface = (String) params.get("publicinterface"); - if (_publicInterface == null) { - throw new ConfigurationException("Unable to find public interface in the configuration parameters"); - } - - _privateInterface = (String) params.get("privateinterface"); - if (_privateInterface == null) { - throw new ConfigurationException("Unable to find private interface in the configuration parameters"); - } - - _numRetries = NumbersUtil.parseInt((String) params.get("numretries"), 2); - - _guid = (String)params.get("guid"); - if (_guid == null) { - throw new ConfigurationException("Unable to find the guid in the configuration parameters"); - } - - _deviceName = (String) params.get("deviceName"); - if (_deviceName == null) { - throw new ConfigurationException("Unable to find the device name in the configuration parameters"); - } - - _isSdx = _deviceName.equalsIgnoreCase("NetscalerSDXLoadBalancer"); - - _inline = Boolean.parseBoolean((String) params.get("inline")); - - if (((String) params.get("cloudmanaged")) != null) { - _cloudManaged = Boolean.parseBoolean((String) params.get("cloudmanaged")); - } - - // validate device configuration parameters - login(); - validateDeviceType(_deviceName); - validateInterfaces(_publicInterface, _privateInterface); - - //enable load balancing feature - enableLoadBalancingFeature(); - - //if the the device is cloud stack provisioned then make it part of the public network - if (_cloudManaged) { - _publicIP = (String) params.get("publicip"); - _publicIPGateway = (String) params.get("publicipgateway"); - _publicIPNetmask = (String) params.get("publicipnetmask"); - _publicIPVlan = (String) params.get("publicipvlan"); - if ("untagged".equalsIgnoreCase(_publicIPVlan)) { - // if public network is un-tagged just add subnet IP - addSubnetIP(_publicIP, _publicIPNetmask); - } else { - // if public network is tagged then add vlan and bind subnet IP to the vlan - addGuestVlanAndSubnet(Long.parseLong(_publicIPVlan), _publicIP, _publicIPNetmask, false); - } - } - - return true; - } catch (Exception e) { - throw new ConfigurationException(e.getMessage()); - } - } - - private void login() throws ExecutionException { - try { - if (!_isSdx) { - _netscalerService = new com.citrix.netscaler.nitro.service.nitro_service(_ip, "https"); - _netscalerService.set_credential(_username, _password); - _netscalerService.set_timeout(_timeout); - apiCallResult = _netscalerService.login(); - if (apiCallResult.errorcode != 0) { - throw new ExecutionException ("Failed to log in to Netscaler device at " + _ip + " due to error " + apiCallResult.errorcode + " and message " + apiCallResult.message); - } - } else { - _netscalerSdxService = new com.citrix.sdx.nitro.service.nitro_service(_ip, "https"); - _netscalerSdxService.set_credential(_username, _password); - com.citrix.sdx.nitro.resource.base.login login = _netscalerSdxService.login(); - if (login == null) { - throw new ExecutionException ("Failed to log in to Netscaler device at " + _ip + " due to error " + apiCallResult.errorcode + " and message " + apiCallResult.message); - } - } - } catch (nitro_exception e) { - throw new ExecutionException("Failed to log in to Netscaler device at " + _ip + " due to " + e.getMessage()); - } catch (Exception e) { - throw new ExecutionException("Failed to log in to Netscaler device at " + _ip + " due to " + e.getMessage()); - } - } - - private void enableLoadBalancingFeature() throws ExecutionException { - if (_isSdx) { - return; - } - try { - String[] features = _netscalerService.get_enabled_features(); - if (features != null) { - for (String feature : features) { - if (feature.equalsIgnoreCase("LB")) { - return; - } - } - } - - // enable load balancing on the device - String[] feature = new String[1]; - feature[0] = "LB"; - apiCallResult = _netscalerService.enable_features(feature); - if (apiCallResult.errorcode != 0) { - throw new ExecutionException("Enabling load balancing feature on the device failed."); - } - } catch (nitro_exception e) { - throw new ExecutionException("Enabling load balancing feature on the device failed due to " + e.getMessage()); - } catch (Exception e) { - throw new ExecutionException("Enabling load balancing feature on the device failed due to " + e.getMessage()); - } - } - - private void validateInterfaces(String publicInterface, String privateInterface) throws ExecutionException { - try { - if (!_isSdx && !_cloudManaged) { - Interface publicIf = Interface.get(_netscalerService, publicInterface); - Interface privateIf = Interface.get(_netscalerService, privateInterface); - if (publicIf != null || privateIf != null) { - return; - } else { - throw new ExecutionException("Invalid interface name specified for public/private interfaces."); - } - } - } catch (nitro_exception e) { - if (e.getErrorCode() == NitroError.NS_RESOURCE_NOT_EXISTS) { - throw new ExecutionException("Invalid interface name specified for public and private interfaces."); - } else { - throw new ExecutionException("Failed to verify public interface and private intefaces are valid due to " + e.getMessage()); - } - } catch (Exception e) { - throw new ExecutionException("Failed to verify public interface and private intefaces are valid due to " + e.getMessage()); - } - } - - private void validateDeviceType(String deviceType) throws ExecutionException { - try { - if (!_isSdx && !_cloudManaged) { - nshardware nsHw = com.citrix.netscaler.nitro.resource.config.ns.nshardware.get(_netscalerService); - if (nsHw == null) { - throw new ExecutionException("Failed to get the hardware description of the Netscaler device at " + _ip); - } else { - if ((_deviceName.equalsIgnoreCase("NetscalerMPXLoadBalancer") && nsHw.get_hwdescription().contains("MPX")) - || (_deviceName.equalsIgnoreCase("NetscalerVPXLoadBalancer") && nsHw.get_hwdescription().contains("NetScaler Virtual Appliance"))) { - return; - } - throw new ExecutionException("Netscalar device type specified does not match with the actuall device type."); - } - } else if (_isSdx) { - mps serviceVM = mps.get(_netscalerSdxService); - if (serviceVM != null) { - if (serviceVM.get_platform().contains("SDX") || serviceVM.get_product().contains("SDX")) { - return; - } else { - throw new ExecutionException("Netscalar device type specified does not match with the actuall device type."); - } - } else { - throw new ExecutionException("Failed to get the hardware details of the Netscaler device at " + _ip); - } - } - } catch (Exception e) { - throw new ExecutionException("Failed to verify device type specified when matching with actuall device type due to " + e.getMessage()); - } - } - - @Override - public StartupCommand[] initialize() { - StartupExternalLoadBalancerCommand cmd = new StartupExternalLoadBalancerCommand(); - cmd.setName(_name); - cmd.setDataCenter(_zoneId); - cmd.setPod(""); - cmd.setPrivateIpAddress(_ip); - cmd.setStorageIpAddress(""); - cmd.setVersion(""); - cmd.setGuid(_guid); - return new StartupCommand[]{cmd}; - } - - @Override - public Answer executeRequest(Command cmd) { - Answer ans = executeRequest(cmd, _numRetries); - return ans; - } - - private Answer executeRequest(Command cmd, int numRetries) { - if (cmd instanceof ReadyCommand) { - return execute((ReadyCommand) cmd); - } else if (cmd instanceof MaintainCommand) { - return execute((MaintainCommand) cmd); - } else if (cmd instanceof IpAssocCommand) { - return execute((IpAssocCommand) cmd, numRetries); - } else if (cmd instanceof LoadBalancerConfigCommand) { - return execute((LoadBalancerConfigCommand) cmd, numRetries); - } else if (cmd instanceof ExternalNetworkResourceUsageCommand) { - return execute((ExternalNetworkResourceUsageCommand) cmd, numRetries); - } else if (cmd instanceof CreateLoadBalancerApplianceCommand) { - return execute((CreateLoadBalancerApplianceCommand) cmd, numRetries); - } else if (cmd instanceof DestroyLoadBalancerApplianceCommand) { - return execute((DestroyLoadBalancerApplianceCommand) cmd, numRetries); - } else if (cmd instanceof SetStaticNatRulesCommand) { - return execute((SetStaticNatRulesCommand) cmd, numRetries); - } else { - return Answer.createUnsupportedCommandAnswer(cmd); - } - } - - private Answer execute(ReadyCommand cmd) { - return new ReadyAnswer(cmd); - } - - protected Answer execute(MaintainCommand cmd) { - return new MaintainAnswer(cmd); - } - - private synchronized Answer execute(IpAssocCommand cmd, int numRetries) { - if (_isSdx) { - return Answer.createUnsupportedCommandAnswer(cmd); - } - - String[] results = new String[cmd.getIpAddresses().length]; - int i = 0; - try { - IpAddressTO[] ips = cmd.getIpAddresses(); - for (IpAddressTO ip : ips) { - long guestVlanTag = Long.valueOf(ip.getVlanId()); - String vlanSelfIp = ip.getVlanGateway(); - String vlanNetmask = ip.getVlanNetmask(); - - if (ip.isAdd()) { - // Add a new guest VLAN and its subnet and bind it to private interface - addGuestVlanAndSubnet(guestVlanTag, vlanSelfIp, vlanNetmask, true); - } else { - // Check and delete guest VLAN with this tag, self IP, and netmask - deleteGuestVlan(guestVlanTag, vlanSelfIp, vlanNetmask); - } - - saveConfiguration(); - results[i++] = ip.getPublicIp() + " - success"; - String action = ip.isAdd() ? "associate" : "remove"; - if (s_logger.isDebugEnabled()) { - s_logger.debug("Netscaler load balancer " + _ip + " successfully executed IPAssocCommand to " + action + " IP " + ip); - } - } - } catch (ExecutionException e) { - s_logger.error("Netscaler loadbalancer " + _ip+ " failed to execute IPAssocCommand due to " + e.getMessage()); - if (shouldRetry(numRetries)) { - return retry(cmd, numRetries); - } else { - results[i++] = IpAssocAnswer.errorResult; - } - } - - return new IpAssocAnswer(cmd, results); - } - - private synchronized Answer execute(LoadBalancerConfigCommand cmd, int numRetries) { - try { - if (_isSdx) { - return Answer.createUnsupportedCommandAnswer(cmd); - } - - LoadBalancerTO[] loadBalancers = cmd.getLoadBalancers(); - if (loadBalancers == null) { - return new Answer(cmd); - } - - for (LoadBalancerTO loadBalancer : loadBalancers) { - String srcIp = loadBalancer.getSrcIp(); - int srcPort = loadBalancer.getSrcPort(); - String lbProtocol = getNetScalerProtocol(loadBalancer); - String lbAlgorithm = loadBalancer.getAlgorithm(); - String nsVirtualServerName = generateNSVirtualServerName(srcIp, srcPort); - - if(loadBalancer.isAutoScaleVmGroupTO()) { - applyAutoScaleConfig(loadBalancer); - return new Answer(cmd); - } - - boolean destinationsToAdd = false; - for (DestinationTO destination : loadBalancer.getDestinations()) { - if (!destination.isRevoked()) { - destinationsToAdd = true; - break; - } - } - - if (!loadBalancer.isRevoked() && destinationsToAdd) { - - // create a load balancing virtual server - addLBVirtualServer(nsVirtualServerName, srcIp, srcPort, lbAlgorithm, lbProtocol, loadBalancer.getStickinessPolicies(), null); - if (s_logger.isDebugEnabled()) { - s_logger.debug("Created load balancing virtual server " + nsVirtualServerName + " on the Netscaler device"); - } - - for (DestinationTO destination : loadBalancer.getDestinations()) { - - String nsServerName = generateNSServerName(destination.getDestIp()); - String nsServiceName = generateNSServiceName(destination.getDestIp(), destination.getDestPort()); - - if (!destination.isRevoked()) { - // add a new destination to deployed load balancing rule - - // add a new server - if (!nsServerExists(nsServerName)) { - com.citrix.netscaler.nitro.resource.config.basic.server nsServer = new com.citrix.netscaler.nitro.resource.config.basic.server(); - nsServer.set_name(nsServerName); - nsServer.set_ipaddress(destination.getDestIp()); - apiCallResult = com.citrix.netscaler.nitro.resource.config.basic.server.add(_netscalerService, nsServer); - if ((apiCallResult.errorcode != 0) && (apiCallResult.errorcode != NitroError.NS_RESOURCE_EXISTS)) { - throw new ExecutionException("Failed to add server " + destination.getDestIp() + " due to" + apiCallResult.message); - } - } - - // create a new service using the server added - if (!nsServiceExists(nsServiceName)) { - com.citrix.netscaler.nitro.resource.config.basic.service newService = new com.citrix.netscaler.nitro.resource.config.basic.service(); - newService.set_name(nsServiceName); - newService.set_port(destination.getDestPort()); - newService.set_servername(nsServerName); - newService.set_state("ENABLED"); - newService.set_servicetype(lbProtocol); - - apiCallResult = com.citrix.netscaler.nitro.resource.config.basic.service.add(_netscalerService, newService); - if (apiCallResult.errorcode != 0) { - throw new ExecutionException("Failed to create service " + nsServiceName + " using server " + nsServerName + " due to" + apiCallResult.message); - } - } - - //bind service to load balancing virtual server - if (!nsServiceBindingExists(nsVirtualServerName, nsServiceName)) { - com.citrix.netscaler.nitro.resource.config.lb.lbvserver_service_binding svcBinding = new com.citrix.netscaler.nitro.resource.config.lb.lbvserver_service_binding(); - svcBinding.set_name(nsVirtualServerName); - svcBinding.set_servicename(nsServiceName); - apiCallResult = com.citrix.netscaler.nitro.resource.config.lb.lbvserver_service_binding.add(_netscalerService, svcBinding); - - if (apiCallResult.errorcode != 0) { - throw new ExecutionException("Failed to bind service: " + nsServiceName + " to the lb virtual server: " + nsVirtualServerName + " on Netscaler device"); - } - } - if (s_logger.isDebugEnabled()) { - s_logger.debug("Successfully added LB destination: " + destination.getDestIp() + ":" + destination.getDestPort() + " to load balancer " + srcIp + ":" + srcPort); - } - } else { - // remove a destination from the deployed load balancing rule - com.citrix.netscaler.nitro.resource.config.lb.lbvserver_service_binding[] serviceBindings = com.citrix.netscaler.nitro.resource.config.lb.lbvserver_service_binding.get(_netscalerService, - nsVirtualServerName); - if (serviceBindings != null) { - for (com.citrix.netscaler.nitro.resource.config.lb.lbvserver_service_binding binding : serviceBindings) { - if (nsServiceName.equalsIgnoreCase(binding.get_servicename())) { - // delete the binding - apiCallResult = com.citrix.netscaler.nitro.resource.config.lb.lbvserver_service_binding.delete(_netscalerService, binding); - if (apiCallResult.errorcode != 0) { - throw new ExecutionException("Failed to delete the binding between the virtual server: " + nsVirtualServerName + " and service:" + nsServiceName + " due to" - + apiCallResult.message); - } - - // check if service is bound to any other virtual server - if (!isServiceBoundToVirtualServer(nsServiceName)) { - // no lb virtual servers are bound to this service so delete it - apiCallResult = com.citrix.netscaler.nitro.resource.config.basic.service.delete(_netscalerService, nsServiceName); - if (apiCallResult.errorcode != 0) { - throw new ExecutionException("Failed to delete service: " + nsServiceName + " due to " + apiCallResult.message); - } - } - - // delete the server if there is no associated services - server_service_binding[] services = server_service_binding.get(_netscalerService, nsServerName); - if ((services == null) || (services.length == 0)) { - apiCallResult = com.citrix.netscaler.nitro.resource.config.basic.server.delete(_netscalerService, nsServerName); - if (apiCallResult.errorcode != 0) { - throw new ExecutionException("Failed to remove server:" + nsServerName + " due to " + apiCallResult.message); - } - } - } - } - } - } - } - } else { - // delete the implemented load balancing rule and its destinations - lbvserver lbserver = getVirtualServerIfExisits(nsVirtualServerName); - if (lbserver != null) { - //unbind the all services associated with this virtual server - com.citrix.netscaler.nitro.resource.config.lb.lbvserver_service_binding[] serviceBindings = com.citrix.netscaler.nitro.resource.config.lb.lbvserver_service_binding.get(_netscalerService, - nsVirtualServerName); - - if (serviceBindings != null) { - for (com.citrix.netscaler.nitro.resource.config.lb.lbvserver_service_binding binding : serviceBindings) { - String serviceName = binding.get_servicename(); - apiCallResult = com.citrix.netscaler.nitro.resource.config.lb.lbvserver_service_binding.delete(_netscalerService, binding); - if (apiCallResult.errorcode != 0) { - throw new ExecutionException("Failed to unbind service from the lb virtual server: " + nsVirtualServerName + " due to " + apiCallResult.message); - } - - com.citrix.netscaler.nitro.resource.config.basic.service svc = com.citrix.netscaler.nitro.resource.config.basic.service.get(_netscalerService, serviceName); - String nsServerName = svc.get_servername(); - - // check if service is bound to any other virtual server - if (!isServiceBoundToVirtualServer(serviceName)) { - // no lb virtual servers are bound to this service so delete it - apiCallResult = com.citrix.netscaler.nitro.resource.config.basic.service.delete(_netscalerService, serviceName); - if (apiCallResult.errorcode != 0) { - throw new ExecutionException("Failed to delete service: " + serviceName + " due to " + apiCallResult.message); - } - } - - //delete the server if no more services attached - server_service_binding[] services = server_service_binding.get(_netscalerService, nsServerName); - if ((services == null) || (services.length == 0)) { - apiCallResult = com.citrix.netscaler.nitro.resource.config.basic.server.delete(_netscalerService, nsServerName); - if (apiCallResult.errorcode != 0) { - throw new ExecutionException("Failed to remove server:" + nsServerName + " due to " + apiCallResult.message); - } - } - } - } - removeLBVirtualServer(nsVirtualServerName); - } - } - } - - if (s_logger.isInfoEnabled()) { - s_logger.info("Successfully executed resource LoadBalancerConfigCommand: " + _gson.toJson(cmd)); - } - - saveConfiguration(); - return new Answer(cmd); - } catch (ExecutionException e) { - s_logger.error("Failed to execute LoadBalancerConfigCommand due to " + e.getMessage()); - if (shouldRetry(numRetries)) { - return retry(cmd, numRetries); - } else { - return new Answer(cmd, e); - } - } catch (Exception e) { - s_logger.error("Failed to execute LoadBalancerConfigCommand due to " + e.getMessage()); - if (shouldRetry(numRetries)) { - return retry(cmd, numRetries); - } else { - return new Answer(cmd, e); - } - } - } - - private synchronized Answer execute(CreateLoadBalancerApplianceCommand cmd, int numRetries) { - - if (!_isSdx) { - return Answer.createUnsupportedCommandAnswer(cmd); - } - - try { - String vpxName = "Cloud-VPX-"+cmd.getLoadBalancerIP(); - String username = "admin"; - String password = "admin"; - - ns ns_obj = new ns(); - ns_obj.set_name(vpxName); - ns_obj.set_ip_address(cmd.getLoadBalancerIP()); - ns_obj.set_netmask(cmd.getNetmask()); - ns_obj.set_gateway(cmd.getGateway()); - ns_obj.set_username(username); - ns_obj.set_password(password); - - // configure VPX instances with defaults - ns_obj.set_feature_license("Standard"); - ns_obj.set_memory_total(new Double(2048)); - ns_obj.set_throughput(new Double(1000)); - ns_obj.set_pps(new Double(1000000)); - ns_obj.set_number_of_ssl_cores(0); - - // use the first device profile available on the SDX to create an instance of VPX - device_profile[] profiles = device_profile.get(_netscalerSdxService); - if (!(profiles != null && profiles.length >= 1)) { - new Answer(cmd, new ExecutionException("Failed to create VPX instance on the netscaler SDX device " + _ip + - " as there are no admin profile to use for creating VPX.")); - } - String profileName = profiles[0].get_name(); - ns_obj.set_nsroot_profile(profileName); - - // use the first VPX image of the available VPX images on the SDX to create an instance of VPX - // TODO: should enable the option to choose the template while adding the SDX device in to CloudStack - xen_vpx_image[] vpxImages = xen_vpx_image.get(_netscalerSdxService); - if (!(vpxImages != null && vpxImages.length >= 1)) { - new Answer(cmd, new ExecutionException("Failed to create VPX instance on the netscaler SDX device " + _ip + - " as there are no VPX images on SDX to use for creating VPX.")); - } - String imageName = vpxImages[0].get_file_name(); - ns_obj.set_image_name(imageName); - - String publicIf = _publicInterface; - String privateIf = _privateInterface; - - // enable only the interfaces that will be used by VPX - enableVPXInterfaces(_publicInterface, _privateInterface, ns_obj); - - // create new VPX instance - ns newVpx = ns.add(_netscalerSdxService, ns_obj); - - if (newVpx == null) { - new Answer(cmd, new ExecutionException("Failed to create VPX instance on the netscaler SDX device " + _ip)); - } - - // wait for VPX instance to start-up - long startTick = System.currentTimeMillis(); - long startWaitMilliSeconds = 600000; - while(!newVpx.get_ns_state().equalsIgnoreCase("up") && System.currentTimeMillis() - startTick < startWaitMilliSeconds) { - try { - Thread.sleep(10000); - } catch(InterruptedException e) { - } - ns refreshNsObj = new ns(); - refreshNsObj.set_id(newVpx.get_id()); - newVpx = ns.get(_netscalerSdxService, refreshNsObj); - } - - // if vpx instance never came up then error out - if (!newVpx.get_ns_state().equalsIgnoreCase("up")) { - return new Answer(cmd, new ExecutionException("Failed to start VPX instance " + vpxName + " created on the netscaler SDX device " + _ip)); - } - - // wait till NS service in side VPX is actually ready - startTick = System.currentTimeMillis(); - boolean nsServiceUp = false; - long nsServiceWaitMilliSeconds = 60000; - while (System.currentTimeMillis() - startTick < nsServiceWaitMilliSeconds) { - try { - nitro_service _netscalerService = new nitro_service(cmd.getLoadBalancerIP(), "https"); - _netscalerService.set_credential(username, password); - _netscalerService.set_timeout(_timeout); - apiCallResult = _netscalerService.login(); - if (apiCallResult.errorcode == 0) { - nsServiceUp = true; - break; - } - } catch (Exception e) { - Thread.sleep(10000); - continue; - } - } - - if (!nsServiceUp) { - return new Answer(cmd, new ExecutionException("Failed to create VPX instance " + vpxName + " on the netscaler SDX device " + _ip)); - } - - if (s_logger.isInfoEnabled()) { - s_logger.info("Successfully provisioned VPX instance " + vpxName + " on the Netscaler SDX device " + _ip); - } - - // physical interfaces on the SDX range from 10/1 to 10/8 & 1/1 to 1/8 of which two different port or same - // port can be used for public and private interfaces - // However the VPX instances created will have interface range start from 10/1 but will only have as many - // interfaces enabled while creating the VPX instance - // So due to this, we need to map public & private interface on SDX to correct public & private interface of - // VPX - - int publicIfnum = Integer.parseInt(_publicInterface.substring(_publicInterface.lastIndexOf("/") + 1)); - int privateIfnum = Integer.parseInt(_privateInterface.substring(_privateInterface.lastIndexOf("/") + 1)); - - if (_publicInterface.startsWith("10/") && _privateInterface.startsWith("10/")) { - if (publicIfnum == privateIfnum) { - publicIf = "10/1"; - privateIf = "10/1"; - } else if (publicIfnum > privateIfnum) { - privateIf = "10/1"; - publicIf = "10/2"; - } else { - publicIf = "10/1"; - privateIf = "10/2"; - } - } else if (_publicInterface.startsWith("1/") && _privateInterface.startsWith("1/")) { - if (publicIfnum == privateIfnum) { - publicIf = "1/1"; - privateIf = "1/1"; - } else if (publicIfnum > privateIfnum) { - privateIf = "1/1"; - publicIf = "1/2"; - } else { - publicIf = "1/1"; - privateIf = "1/2"; - } - } else if (_publicInterface.startsWith("1/") && _privateInterface.startsWith("10/")) { - publicIf = "1/1"; - privateIf = "10/1"; - } else if (_publicInterface.startsWith("10/") && _privateInterface.startsWith("1/")) { - publicIf = "10/1"; - privateIf = "1/1"; - } - - return new CreateLoadBalancerApplianceAnswer(cmd, true, "provisioned VPX instance", "NetscalerVPXLoadBalancer", "Netscaler", new NetscalerResource(), - publicIf, privateIf, _username, _password); - } catch (Exception e) { - if (shouldRetry(numRetries)) { - return retry(cmd, numRetries); - } - return new CreateLoadBalancerApplianceAnswer(cmd, false, "failed to provisioned VPX instance due to " + e.getMessage(), null, null, null, null, null, null, null); - } - } - - private void enableVPXInterfaces(String publicIf, String privateIf, ns ns_obj) { - // enable VPX to use 10 gigabit Ethernet interfaces if public/private interface - // on SDX is a 10Gig interface - if (publicIf.equals("10/1") || privateIf.equals("10/1")) { - ns_obj.set_if_10_1(new Boolean(true)); - } - - if (publicIf.equals("10/2") || privateIf.equals("10/2")) { - ns_obj.set_if_10_2(new Boolean(true)); - } - - if (publicIf.equals("10/3") || privateIf.equals("10/3")) { - ns_obj.set_if_10_3(new Boolean(true)); - } - - if (publicIf.equals("10/4") || privateIf.equals("10/4")) { - ns_obj.set_if_10_4(new Boolean(true)); - } - - if (publicIf.equals("10/5") || privateIf.equals("10/5")) { - ns_obj.set_if_10_5(new Boolean(true)); - } - - if (publicIf.equals("10/6") || privateIf.equals("10/6")) { - ns_obj.set_if_10_6(new Boolean(true)); - } - - if (publicIf.equals("10/7") || privateIf.equals("10/7")) { - ns_obj.set_if_10_7(new Boolean(true)); - } - - if (publicIf.equals("10/8") || privateIf.equals("10/8")) { - ns_obj.set_if_10_8(new Boolean(true)); - } - - // enable VPX to use 1 gigabit Ethernet interfaces if public/private interface - // on SDX is a 1Gig interface - if (publicIf.equals("1/1") || privateIf.equals("1/1")) { - ns_obj.set_if_1_1(new Boolean(true)); - } - - if (publicIf.equals("1/2") || privateIf.equals("1/2")) { - ns_obj.set_if_1_2(new Boolean(true)); - } - - if (publicIf.equals("1/3") || privateIf.equals("1/3")) { - ns_obj.set_if_1_3(new Boolean(true)); - } - - if (publicIf.equals("1/4") || privateIf.equals("1/4")) { - ns_obj.set_if_1_4(new Boolean(true)); - } - - if (publicIf.equals("1/5") || privateIf.equals("1/5")) { - ns_obj.set_if_1_5(new Boolean(true)); - } - - if (publicIf.equals("1/6") || privateIf.equals("1/6")) { - ns_obj.set_if_1_6(new Boolean(true)); - } - - if (publicIf.equals("1/7") || privateIf.equals("1/7")) { - ns_obj.set_if_1_7(new Boolean(true)); - } - - if (publicIf.equals("1/8") || privateIf.equals("1/8")) { - ns_obj.set_if_1_8(new Boolean(true)); - } - } - - private synchronized Answer execute(DestroyLoadBalancerApplianceCommand cmd, int numRetries) { - String vpxName = "Cloud-VPX-"+cmd.getLoadBalancerIP(); - if (!_isSdx) { - return Answer.createUnsupportedCommandAnswer(cmd); - } - - try { - ns vpxToDelete =null; - ns[] vpxInstances = ns.get(_netscalerSdxService); - for (ns vpx : vpxInstances) { - if (vpx.get_name().equals(vpxName)) { - vpxToDelete = vpx; - break; - } - } - - if (vpxToDelete == null) { - String msg = "There is no VPX instance " + vpxName + " on the Netscaler SDX device " + _ip + " to delete"; - s_logger.warn(msg); - return new DestroyLoadBalancerApplianceAnswer(cmd, true, msg); - } - - // destroy the VPX instance - ns nsDelObj = new ns(); - nsDelObj.set_id(vpxToDelete.get_id()); - vpxToDelete = ns.delete(_netscalerSdxService, nsDelObj); - String msg = "Deleted VPX instance " + vpxName + " on Netscaler SDX " + _ip + " successfully."; - s_logger.info(msg); - return new DestroyLoadBalancerApplianceAnswer(cmd, true,msg); - } catch (Exception e) { - if (shouldRetry(numRetries)) { - return retry(cmd, numRetries); - } - return new DestroyLoadBalancerApplianceAnswer(cmd, false, "Failed to delete VPX instance " + vpxName + " on Netscaler SDX " + _ip); - } - } - - private synchronized Answer execute(SetStaticNatRulesCommand cmd, int numRetries) { - - if (_isSdx) { - return Answer.createUnsupportedCommandAnswer(cmd); - } - - String[] results = new String[cmd.getRules().length]; - int i = 0; - boolean endResult = true; - - try { - for (StaticNatRuleTO rule : cmd.getRules()) { - String srcIp = rule.getSrcIp(); - String dstIP = rule.getDstIp(); - String iNatRuleName = generateInatRuleName(srcIp, dstIP); - inat iNatRule = null; - - if (!rule.revoked()) { - try { - iNatRule = inat.get(_netscalerService, iNatRuleName); - } catch (nitro_exception e) { - if (e.getErrorCode() != NitroError.NS_RESOURCE_NOT_EXISTS) { - throw e; - } - } - - if (iNatRule == null) { - iNatRule = new inat(); - iNatRule.set_name(iNatRuleName); - iNatRule.set_publicip(srcIp); - iNatRule.set_privateip(dstIP); - iNatRule.set_usnip("OFF"); - iNatRule.set_usip("ON"); - try { - apiCallResult = inat.add(_netscalerService, iNatRule); - } catch (nitro_exception e) { - if (e.getErrorCode() != NitroError.NS_RESOURCE_EXISTS) { - throw e; - } - } - s_logger.debug("Created Inat rule on the Netscaler device " + _ip + " to enable static NAT from " + srcIp + " to " + dstIP); - } - } else { - try { - inat.delete(_netscalerService, iNatRuleName); - } catch (nitro_exception e) { - if (e.getErrorCode() != NitroError.NS_RESOURCE_NOT_EXISTS) { - throw e; - } - } - s_logger.debug("Deleted Inat rule on the Netscaler device " + _ip + " to remove static NAT from " + srcIp + " to " + dstIP); - } - - saveConfiguration(); - results[i++] = "Static nat rule from " + srcIp + " to " + dstIP + " successfully " + (rule.revoked() ? " revoked.":" created."); - } - } catch (Exception e) { - if (shouldRetry(numRetries)) { - return retry(cmd, numRetries); - } - results[i++] = "Configuring static nat rule failed due to " + e.getMessage(); - endResult = false; - return new SetStaticNatRulesAnswer(cmd, results, endResult); - } - - return new SetStaticNatRulesAnswer(cmd, results, endResult); - } - - private synchronized Answer execute(ExternalNetworkResourceUsageCommand cmd, int numRetries) { - try { - if (!_isSdx) { - return getPublicIpBytesSentAndReceived(cmd); - } else { - return Answer.createUnsupportedCommandAnswer(cmd); - } - } catch (ExecutionException e) { - if (shouldRetry(numRetries)) { - return retry(cmd, numRetries); - } else { - return new ExternalNetworkResourceUsageAnswer(cmd, e); - } - } - } - - private void addSubnetIP(String snip, String netmask) throws ExecutionException { - try { - nsip selfIp = new nsip(); - selfIp.set_ipaddress(snip); - selfIp.set_netmask(netmask); - selfIp.set_type("SNIP"); - apiCallResult = nsip.add(_netscalerService, selfIp); - if (apiCallResult.errorcode != 0) { - throw new ExecutionException("Failed to add SNIP object on the Netscaler device due to "+ apiCallResult.message); - } - } catch (nitro_exception e) { - throw new ExecutionException("Failed to add SNIP object on the Netscaler device due to " + e.getMessage()); - } catch (Exception e) { - throw new ExecutionException("Failed to add SNIP object on the Netscaler device due to " + e.getMessage()); - } - } - - private void addGuestVlanAndSubnet(long vlanTag, String vlanSelfIp, String vlanNetmask, boolean guestVlan) throws ExecutionException { - try { - // add vlan object for guest VLAN - if (!nsVlanExists(vlanTag)) { - try { - vlan vlanObj = new vlan(); - vlanObj.set_id(vlanTag); - apiCallResult = vlan.add(_netscalerService, vlanObj); - if (apiCallResult.errorcode != 0) { - throw new ExecutionException("Failed to add new vlan with tag:" + vlanTag + "on the NetScaler device due to " + apiCallResult.message); - } - } catch (nitro_exception e) { - throw new ExecutionException("Failed to add new vlan with tag:" + vlanTag + "on the NetScaler device due to " + e.getMessage()); - } - } - - // add subnet IP object for this guest network - if (!nsSnipExists(vlanSelfIp)) { - try { - nsip selfIp = new nsip(); - selfIp.set_ipaddress(vlanSelfIp); - selfIp.set_netmask(vlanNetmask); - selfIp.set_type("SNIP"); - apiCallResult = nsip.add(_netscalerService, selfIp); - if (apiCallResult.errorcode != 0) { - throw new ExecutionException("Failed to add SNIP object for the guest network on the Netscaler device due to "+ apiCallResult.message); - } - } catch (nitro_exception e) { - throw new ExecutionException("Failed to add SNIP object for the guest network on the Netscaler device due to " + e.getMessage()); - } - } - - // bind the vlan object to subnet IP object - if (!nsVlanNsipBindingExists(vlanTag, vlanSelfIp)) { - try { - vlan_nsip_binding ipVlanBinding = new vlan_nsip_binding(); - ipVlanBinding.set_id(vlanTag); - ipVlanBinding.set_ipaddress(vlanSelfIp); - ipVlanBinding.set_netmask(vlanNetmask); - apiCallResult = vlan_nsip_binding.add(_netscalerService, ipVlanBinding); - if (apiCallResult.errorcode != 0) { - throw new ExecutionException("Failed to bind VLAN with tag:" + vlanTag + " to the subnet due to " + apiCallResult.message); - } - } catch (nitro_exception e) { - throw new ExecutionException("Failed to bind VLAN with tage:"+ vlanTag + " to the subnet due to " + e.getMessage()); - } - } - - // bind vlan object to the private interface - try { - vlan_interface_binding vlanBinding = new vlan_interface_binding(); - if (guestVlan) { - vlanBinding.set_ifnum(_privateInterface); - } else { - vlanBinding.set_ifnum(_publicInterface); - } - vlanBinding.set_tagged(true); - vlanBinding.set_id(vlanTag); - apiCallResult = vlan_interface_binding.add(_netscalerService, vlanBinding); - if (apiCallResult.errorcode != 0) { - String vlanInterface = guestVlan ? _privateInterface : _publicInterface; - throw new ExecutionException("Failed to bind vlan with tag:" + vlanTag + " with the interface " + vlanInterface + " due to " + apiCallResult.message); - } - } catch (nitro_exception e) { - if (!(e.getErrorCode() == NitroError.NS_INTERFACE_ALREADY_BOUND_TO_VLAN)) { - throw new ExecutionException("Failed to bind VLAN "+ vlanTag + " with interface on the Netscaler device due to " + e.getMessage()); - } - } - } catch (nitro_exception e) { - throw new ExecutionException("Failed to implement guest network on the Netscaler device due to " + e.getMessage()); - } catch (Exception e) { - throw new ExecutionException("Failed to implement guest network on the Netscaler device due to " + e.getMessage()); - } - } - - private void deleteGuestVlan(long vlanTag, String vlanSelfIp, String vlanNetmask) throws ExecutionException { - try { - - // Delete all servers and associated services from this guest VLAN - deleteServersInGuestVlan(vlanTag, vlanSelfIp, vlanNetmask); - - // unbind vlan to the private interface - try { - vlan_interface_binding vlanIfBinding = new vlan_interface_binding(); - vlanIfBinding.set_id(vlanTag); - vlanIfBinding.set_ifnum(_privateInterface); - vlanIfBinding.set_tagged(true); - apiCallResult = vlan_interface_binding.delete(_netscalerService, vlanIfBinding); - if (apiCallResult.errorcode != 0) { - throw new ExecutionException("Failed to unbind vlan:" + vlanTag + " with the private interface due to " + apiCallResult.message); - } - } catch (nitro_exception e) { - // if Vlan to interface binding does not exist then ignore the exception and proceed - if (!(e.getErrorCode() == NitroError.NS_RESOURCE_NOT_EXISTS)) { - throw new ExecutionException("Failed to unbind vlan from the interface while shutdown of guest network on the Netscaler device due to " + e.getMessage()); - } - } - - // unbind the vlan to subnet - try { - vlan_nsip_binding vlanSnipBinding = new vlan_nsip_binding(); - vlanSnipBinding.set_netmask(vlanNetmask); - vlanSnipBinding.set_ipaddress(vlanSelfIp); - vlanSnipBinding.set_id(vlanTag); - apiCallResult = vlan_nsip_binding.delete(_netscalerService, vlanSnipBinding); - if (apiCallResult.errorcode != 0) { - throw new ExecutionException("Failed to unbind vlan:" + vlanTag + " with the subnet due to " + apiCallResult.message); - } - } catch (nitro_exception e) { - // if Vlan to subnet binding does not exist then ignore the exception and proceed - if (!(e.getErrorCode() == NitroError.NS_RESOURCE_NOT_EXISTS)) { - throw new ExecutionException("Failed to unbind vlan:" + vlanTag + " with the subnet due to " + e.getMessage()); - } - } - - // remove subnet IP - try { - nsip subnetIp = nsip.get(_netscalerService, vlanSelfIp); - apiCallResult = nsip.delete(_netscalerService, subnetIp); - if (apiCallResult.errorcode != 0) { - throw new ExecutionException("Failed to remove subnet ip:" + vlanSelfIp + " from the NetScaler device due to" + apiCallResult.message); - } - } catch (nitro_exception e) { - // if subnet SNIP does not exist then ignore the exception and proceed - if (!(e.getErrorCode() == NitroError.NS_RESOURCE_NOT_EXISTS)) { - throw new ExecutionException("Failed to remove subnet ip:" + vlanSelfIp + " from the NetScaler device due to" + e.getMessage()); - } - } - - // remove the vlan from the NetScaler device - if (nsVlanExists(vlanTag)) { - // remove vlan - apiCallResult = com.citrix.netscaler.nitro.resource.config.network.vlan.delete(_netscalerService, vlanTag); - if (apiCallResult.errorcode != 0) { - throw new ExecutionException("Failed to remove vlan with tag:" + vlanTag + "due to" + apiCallResult.message); - } - } - } catch (nitro_exception e) { - throw new ExecutionException("Failed to delete guest vlan network on the Netscaler device due to " + e.getMessage()); - } catch (Exception e) { - throw new ExecutionException("Failed to delete guest vlan network on the Netscaler device due to " + e.getMessage()); - } - } - - private boolean nsVlanExists(long vlanTag) throws ExecutionException { - try { - if (vlan.get(_netscalerService, new Long(vlanTag)) != null) { - return true; - } else { - return false; - } - } catch (nitro_exception e) { - if (e.getErrorCode() == NitroError.NS_RESOURCE_NOT_EXISTS) { - return false; - } else { - throw new ExecutionException("Failed to verify VLAN exists on the NetScaler device due to " + e.getMessage()); - } - } catch (Exception e) { - throw new ExecutionException("Failed to verify VLAN exists on the NetScaler device due to " + e.getMessage()); - } - } - - private boolean nsSnipExists(String subnetIP) throws ExecutionException { - try { - nsip snip = nsip.get(_netscalerService, subnetIP); - return (snip != null); - } catch (nitro_exception e) { - if (e.getErrorCode() == NitroError.NS_RESOURCE_NOT_EXISTS) { - return false; - } else { - throw new ExecutionException("Failed to verify if SNIP exists on the NetScaler device due to " + e.getMessage()); - } - } catch (Exception e) { - throw new ExecutionException("Failed to verify if SNIP exists on the NetScaler device due to " + e.getMessage()); - } - } - - private boolean nsServerExists(String serverName) throws ExecutionException { - try { - if (com.citrix.netscaler.nitro.resource.config.basic.server.get(_netscalerService, serverName) != null) { - return true; - } else { - return false; - } - } catch (nitro_exception e) { - if (e.getErrorCode() == NitroError.NS_RESOURCE_NOT_EXISTS) { - return false; - } else { - throw new ExecutionException("Failed to verify Server " + serverName + " exists on the NetScaler device due to " + e.getMessage()); - } - } catch (Exception e) { - throw new ExecutionException("Failed to verify Server " + serverName + " exists on the NetScaler device due to " + e.getMessage()); - } - } - - private boolean nsVlanNsipBindingExists(long vlanTag, String vlanSelfIp) throws ExecutionException { - try { - vlan_nsip_binding[] vlanNsipBindings = vlan_nsip_binding.get(_netscalerService, vlanTag); - if (vlanNsipBindings != null && vlanNsipBindings[0] != null && vlanNsipBindings[0].get_ipaddress().equalsIgnoreCase(vlanSelfIp)) { - return true; - } else { - return false; - } - } catch (nitro_exception e) { - if (e.getErrorCode() == NitroError.NS_RESOURCE_NOT_EXISTS) { - return false; - } else { - throw new ExecutionException("Failed to verify Vlan " + vlanTag + " to SNIP " + vlanSelfIp + " binding exists due to " + e.getMessage()); - } - } catch (Exception e) { - throw new ExecutionException("Failed to verify Vlan " + vlanTag + " to SNIP " + vlanSelfIp + " binding exists due to " + e.getMessage()); - } - } - - private lbvserver getVirtualServerIfExisits(String lbVServerName ) throws ExecutionException { - try { - return lbvserver.get(_netscalerService, lbVServerName); - } catch (nitro_exception e) { - if (e.getErrorCode() == NitroError.NS_RESOURCE_NOT_EXISTS) { - return null; - } else { - throw new ExecutionException(e.getMessage()); - } - } catch (Exception e) { - throw new ExecutionException(e.getMessage()); - } - } - - private boolean isServiceBoundToVirtualServer(String serviceName) throws ExecutionException { - try { - lbvserver[] lbservers = lbvserver.get(_netscalerService); - for (lbvserver vserver : lbservers) { - filtervalue[] filter = new filtervalue[1]; - filter[0] = new filtervalue("servicename", serviceName); - lbvserver_service_binding[] result = lbvserver_service_binding.get_filtered(_netscalerService, vserver.get_name(), filter); - if (result != null && result.length > 0) { - return true; - } - } - return false; - } catch (Exception e) { - throw new ExecutionException("Failed to verify service " + serviceName + " is bound to any virtual server due to " + e.getMessage()); - } - } - - private boolean nsServiceExists(String serviceName) throws ExecutionException { - try { - if (com.citrix.netscaler.nitro.resource.config.basic.service.get(_netscalerService, serviceName) != null) { - return true; - } else { - return false; - } - } catch (nitro_exception e) { - if (e.getErrorCode() == NitroError.NS_NO_SERIVCE) { - return false; - } else { - throw new ExecutionException("Failed to verify service " + serviceName + " exists due to " + e.getMessage()); - } - } catch (Exception e) { - throw new ExecutionException("Failed to verify service " + serviceName + " exists due to " + e.getMessage()); - } - } - - private boolean nsServiceBindingExists(String lbVirtualServer, String serviceName) throws ExecutionException { - try { - com.citrix.netscaler.nitro.resource.config.lb.lbvserver_service_binding[] serviceBindings = com.citrix.netscaler.nitro.resource.config.lb.lbvserver_service_binding.get(_netscalerService, lbVirtualServer); - if (serviceBindings != null) { - for (com.citrix.netscaler.nitro.resource.config.lb.lbvserver_service_binding binding : serviceBindings) { - if (serviceName.equalsIgnoreCase(binding.get_servicename())) { - return true; - } - } - } - return false; - } catch (nitro_exception e) { - throw new ExecutionException("Failed to verify lb vserver " + lbVirtualServer + "and service " + serviceName + " binding exists due to " + e.getMessage()); - } catch (Exception e) { - throw new ExecutionException("Failed to verify lb vserver " + lbVirtualServer + "and service " + serviceName + " binding exists due to " + e.getMessage()); - } - } - - private boolean isServiceGroupBoundToVirtualServer(String nsVirtualServerName, String serviceGroupName) throws ExecutionException { - - lbvserver_servicegroup_binding vserver_servicegroup_binding = new lbvserver_servicegroup_binding(); - - try { - lbvserver_servicegroup_binding[] result = vserver_servicegroup_binding.get_filtered(_netscalerService, nsVirtualServerName, "servicegroupname:" + serviceGroupName); - if(result != null && result.length > 0){ - return true; - } - } catch (Exception e) { - throw new ExecutionException("Failed to verify lb vserver " + nsVirtualServerName + "and servicegrop " + serviceGroupName + " binding exists due to " + e.getMessage()); - } - return false; - - } - - private servicegroup getServiceGroupIfExisits(String lbVServerName ) throws ExecutionException { - try { - return servicegroup.get(_netscalerService, lbVServerName); - } catch (nitro_exception e) { - if (e.getErrorCode() == NitroError.NS_RESOURCE_NOT_EXISTS) { - return null; - } else { - throw new ExecutionException(e.getMessage()); - } - } catch (Exception e) { - throw new ExecutionException(e.getMessage()); - } - } - - private void deleteServersInGuestVlan(long vlanTag, String vlanSelfIp, String vlanNetmask) throws ExecutionException { - try { - com.citrix.netscaler.nitro.resource.config.basic.server[] serverList = com.citrix.netscaler.nitro.resource.config.basic.server.get(_netscalerService); - - if (serverList == null) { - return; - } - - // remove the server and services associated with guest vlan - for (com.citrix.netscaler.nitro.resource.config.basic.server server : serverList) { - // check if server belong to same subnet as one associated with vlan - if (NetUtils.sameSubnet(vlanSelfIp, server.get_ipaddress(), vlanNetmask)) { - // first remove services associated with this server - com.citrix.netscaler.nitro.resource.config.basic.service serveicesList[] = com.citrix.netscaler.nitro.resource.config.basic.service.get(_netscalerService); - if (serveicesList != null) { - for (com.citrix.netscaler.nitro.resource.config.basic.service svc : serveicesList) { - if (svc.get_servername().equals(server.get_ipaddress())) { - apiCallResult = com.citrix.netscaler.nitro.resource.config.basic.service.delete(_netscalerService, svc.get_name()); - if (apiCallResult.errorcode != 0) { - throw new ExecutionException("Failed to remove service:" + svc.get_name()); - } - } - } - } - // remove the server - apiCallResult = com.citrix.netscaler.nitro.resource.config.basic.server.delete(_netscalerService, server.get_name()); - if (apiCallResult.errorcode != 0) { - throw new ExecutionException("Failed to remove server:" + server.get_name()); - } - } - } - } catch (Exception e) { - throw new ExecutionException("Failed to delete server and services in the guest vlan:" + vlanTag + " on the Netscaler device due to: "+ e.getMessage()); - } - } - - private String getNetScalerProtocol(LoadBalancerTO loadBalancer) throws ExecutionException { - String port = Integer.toString(loadBalancer.getSrcPort()); - String lbProtocol = loadBalancer.getProtocol(); - StickinessPolicyTO[] stickyPolicies = loadBalancer.getStickinessPolicies(); - String nsProtocol = "TCP"; - - if ((stickyPolicies != null) && (stickyPolicies.length > 0) && (stickyPolicies[0] != null)){ - StickinessPolicyTO stickinessPolicy = stickyPolicies[0]; - if (StickinessMethodType.LBCookieBased.getName().equalsIgnoreCase(stickinessPolicy.getMethodName()) || - (StickinessMethodType.AppCookieBased.getName().equalsIgnoreCase(stickinessPolicy.getMethodName()))) { - nsProtocol = "HTTP"; - return nsProtocol; - } - } - - if (port.equals(NetUtils.HTTP_PORT)) { - nsProtocol = "HTTP"; - } else if (NetUtils.TCP_PROTO.equalsIgnoreCase(lbProtocol)) { - nsProtocol = "TCP"; - } else if (NetUtils.UDP_PROTO.equalsIgnoreCase(lbProtocol)) { - nsProtocol = "UDP"; - } - - return nsProtocol; - } - - private void addLBVirtualServer(String virtualServerName, String publicIp, int publicPort, String lbAlgorithm, String protocol, StickinessPolicyTO[] stickyPolicies, AutoScaleVmGroupTO vmGroupTO) throws ExecutionException { - try { - String lbMethod; - if ("roundrobin".equalsIgnoreCase(lbAlgorithm)) { - lbMethod = "ROUNDROBIN"; - } else if ("leastconn".equalsIgnoreCase(lbAlgorithm)) { - lbMethod = "LEASTCONNECTION"; - } else if ("source".equalsIgnoreCase(lbAlgorithm)) { - lbMethod = "SOURCEIPHASH"; - } else { - throw new ExecutionException("Got invalid load balancing algorithm: " + lbAlgorithm + " in the load balancing rule"); - } - - boolean vserverExisis = false; - lbvserver vserver = getVirtualServerIfExisits(virtualServerName); - if (vserver != null) { - if (!vserver.get_servicetype().equalsIgnoreCase(protocol)) { - throw new ExecutionException("Can not update virtual server:" + virtualServerName + " as current protocol:" + vserver.get_servicetype() + " of virtual server is different from the " - + " intended protocol:" + protocol); - } - vserverExisis = true; - } - // Use new vserver always for configuration - vserver = new lbvserver(); - vserver.set_name(virtualServerName); - vserver.set_ipv46(publicIp); - vserver.set_port(publicPort); - vserver.set_servicetype(protocol); - vserver.set_lbmethod(lbMethod); - - // netmask can only be set for source IP load balancer algorithm - if (!lbMethod.equalsIgnoreCase("SOURCEIPHASH")) { - vserver.set_netmask(null); - vserver.set_v6netmasklen(null); - } - - if ((stickyPolicies != null) && (stickyPolicies.length > 0) && (stickyPolicies[0] != null)){ - long timeout = 2;// netscaler default 2 min - String cookieName = null; - StickinessPolicyTO stickinessPolicy = stickyPolicies[0]; - - // get the session persistence parameters - List> paramsList = stickinessPolicy.getParams(); - for(Pair param : paramsList) { - if ("holdtime".equalsIgnoreCase(param.first())) { - timeout = Long.parseLong(param.second()); - } else if ("name".equalsIgnoreCase(param.first())) { - cookieName = param.second(); - } - } - - // configure virtual server based on the persistence method - if (StickinessMethodType.LBCookieBased.getName().equalsIgnoreCase(stickinessPolicy.getMethodName())) { - vserver.set_persistencetype("COOKIEINSERT"); - } else if (StickinessMethodType.SourceBased.getName().equalsIgnoreCase(stickinessPolicy.getMethodName())) { - vserver.set_persistencetype("SOURCEIP"); - } else if (StickinessMethodType.AppCookieBased.getName().equalsIgnoreCase(stickinessPolicy.getMethodName())) { - vserver.set_persistencetype("RULE"); - vserver.set_rule("HTTP.REQ.HEADER(\"COOKIE\").VALUE(0).typecast_nvlist_t('=',';').value(\"" + cookieName + "\")"); - vserver.set_resrule("HTTP.RES.HEADER(\"SET-COOKIE\").VALUE(0).typecast_nvlist_t('=',';').value(\"" + cookieName + "\")"); - } else { - throw new ExecutionException("Got invalid session persistence method: " + stickinessPolicy.getMethodName() + " in the load balancing rule"); - } - - // set session persistence timeout - vserver.set_timeout(timeout); - } else { - // delete the LB stickyness policy - vserver.set_persistencetype("NONE"); - } - - - if (vserverExisis) { - apiCallResult = lbvserver.update(_netscalerService,vserver); - } else { - apiCallResult = lbvserver.add(_netscalerService,vserver); - } - if (apiCallResult.errorcode != 0) { - throw new ExecutionException("Failed to create new load balancing virtual server:" + virtualServerName + " due to " + apiCallResult.message); - } - - if (s_logger.isDebugEnabled()) { - s_logger.debug("Created load balancing virtual server " + virtualServerName + " on the Netscaler device"); - } - } catch (nitro_exception e) { - s_logger.warn("Failed to create virtual server " , e); - throw new ExecutionException("Failed to create new virtual server:" + virtualServerName + " due to " + e.getMessage()); - } catch (Exception e) { - throw new ExecutionException("Failed to create new virtual server:" + virtualServerName + " due to " + e.getMessage()); - } - } - - private void removeLBVirtualServer (String virtualServerName) throws ExecutionException { - try { - lbvserver vserver = lbvserver.get(_netscalerService, virtualServerName); - if (vserver == null) { - return; - } - apiCallResult = lbvserver.delete(_netscalerService, vserver); - if (apiCallResult.errorcode != 0) { - throw new ExecutionException("Failed to delete virtual server:" + virtualServerName + " due to " + apiCallResult.message); - } - } catch (nitro_exception e) { - if (e.getErrorCode() == NitroError.NS_RESOURCE_NOT_EXISTS) { - return; - } else { - throw new ExecutionException("Failed remove virtual server:" + virtualServerName +" due to " + e.getMessage()); - } - } catch (Exception e) { - throw new ExecutionException("Failed to remove virtual server:" + virtualServerName +" due to " + e.getMessage()); - } - } - - public synchronized void applyAutoScaleConfig(LoadBalancerTO loadBalancer) throws Exception, ExecutionException { - - AutoScaleVmGroupTO vmGroupTO = loadBalancer.getAutoScaleVmGroupTO(); - if(!isAutoScaleSupportedInNetScaler()) { - throw new ExecutionException("AutoScale not supported in this version of NetScaler"); - } - if(vmGroupTO.getState().equals("new")) { - assert !loadBalancer.isRevoked(); - createAutoScaleConfig(loadBalancer); - } - else if(loadBalancer.isRevoked() || vmGroupTO.getState().equals("revoke")) { - removeAutoScaleConfig(loadBalancer); - } - else if(vmGroupTO.getState().equals("enabled")) { - assert !loadBalancer.isRevoked(); - enableAutoScaleConfig(loadBalancer, false); - } - else if(vmGroupTO.getState().equals("disabled")) { - assert !loadBalancer.isRevoked(); - disableAutoScaleConfig(loadBalancer, false); - } else { - ///// This should never happen - throw new ExecutionException("Unknown vmGroup State :" + vmGroupTO.getState()); - } - } - - @SuppressWarnings("static-access") - private synchronized boolean createAutoScaleConfig(LoadBalancerTO loadBalancerTO) throws ExecutionException, Exception { - - String srcIp = loadBalancerTO.getSrcIp(); - int srcPort = loadBalancerTO.getSrcPort(); - String lbProtocol = getNetScalerProtocol(loadBalancerTO); - String lbAlgorithm = loadBalancerTO.getAlgorithm(); - String nsVirtualServerName = generateNSVirtualServerName(srcIp, srcPort); - AutoScaleVmGroupTO vmGroupTO = loadBalancerTO.getAutoScaleVmGroupTO(); - if (s_logger.isDebugEnabled()) { - s_logger.debug("Created load balancing virtual server " + nsVirtualServerName + " on the Netscaler device"); - } - addLBVirtualServer(nsVirtualServerName, srcIp, srcPort, lbAlgorithm, lbProtocol, loadBalancerTO.getStickinessPolicies(), vmGroupTO); - - String serviceGroupName = generateAutoScaleServiceGroupName(srcIp, srcPort); - servicegroup serviceGroup = getServiceGroupIfExisits(serviceGroupName); - if(serviceGroup == null) { - // add servicegroup lb_autoscaleGroup -autoscale POLICY -memberPort 80 - int memberPort = vmGroupTO.getMemberPort(); - try { - serviceGroup = new servicegroup(); - serviceGroup.set_servicegroupname(serviceGroupName); - serviceGroup.set_servicetype(lbProtocol); - serviceGroup.set_autoscale("POLICY"); - serviceGroup.set_memberport(memberPort); - serviceGroup.add(_netscalerService, serviceGroup); - } catch (Exception e) { - throw e; - } - } - - if(!isServiceGroupBoundToVirtualServer(nsVirtualServerName, serviceGroupName)) { - // Bind autoscale service group - // bind lb vserver lb lb_autoscaleGroup - lbvserver_servicegroup_binding vserver_servicegroup_binding = new lbvserver_servicegroup_binding(); - - try { - vserver_servicegroup_binding.set_name(nsVirtualServerName); - vserver_servicegroup_binding.set_servicegroupname(serviceGroupName); - vserver_servicegroup_binding.add(_netscalerService, vserver_servicegroup_binding); - } catch (Exception e) { - throw e; - } - } - - // Create the autoscale config - enableAutoScaleConfig(loadBalancerTO, false); - return true; - } - - @SuppressWarnings("static-access") - private synchronized boolean removeAutoScaleConfig(LoadBalancerTO loadBalancerTO) throws Exception, ExecutionException { - String srcIp = loadBalancerTO.getSrcIp(); - int srcPort = loadBalancerTO.getSrcPort(); - - String nsVirtualServerName = generateNSVirtualServerName(srcIp, srcPort); - String serviceGroupName = generateAutoScaleServiceGroupName(srcIp, srcPort); - - if(loadBalancerTO.getAutoScaleVmGroupTO().getState().equals("enabled")) { - disableAutoScaleConfig(loadBalancerTO, false); - } - - if(isServiceGroupBoundToVirtualServer(nsVirtualServerName, serviceGroupName)) { - // UnBind autoscale service group - // unbind lb vserver lb lb_autoscaleGroup - lbvserver_servicegroup_binding vserver_servicegroup_binding = new lbvserver_servicegroup_binding(); - try { - vserver_servicegroup_binding.set_name(nsVirtualServerName); - vserver_servicegroup_binding.set_servicegroupname(serviceGroupName); - vserver_servicegroup_binding.delete(_netscalerService, vserver_servicegroup_binding); - } catch (Exception e) { - throw e; - } - } - - if(getServiceGroupIfExisits(serviceGroupName) != null) { - // Remove autoscale service group - com.citrix.netscaler.nitro.resource.config.basic.servicegroup serviceGroup = new com.citrix.netscaler.nitro.resource.config.basic.servicegroup(); - try { - serviceGroup.set_servicegroupname(serviceGroupName); - serviceGroup.delete(_netscalerService, serviceGroup); - } catch (Exception e) { - throw e; - } - } - - removeLBVirtualServer(nsVirtualServerName); - - return true; - } - - @SuppressWarnings("static-access") - private synchronized boolean enableAutoScaleConfig(LoadBalancerTO loadBalancerTO, boolean isCleanUp) throws Exception { - String srcIp = loadBalancerTO.getSrcIp(); - int srcPort = loadBalancerTO.getSrcPort(); - - String nsVirtualServerName = generateNSVirtualServerName(srcIp, srcPort); - String serviceGroupName = generateAutoScaleServiceGroupName(srcIp, srcPort); - String profileName = generateAutoScaleProfileName(srcIp, srcPort); - String timerName = generateAutoScaleTimerName(srcIp, srcPort); - String scaleDownActionName = generateAutoScaleScaleDownActionName(srcIp, srcPort); - String scaleUpActionName = generateAutoScaleScaleUpActionName(srcIp, srcPort); - String mtName = generateSnmpMetricTableName(srcIp, srcPort); - String monitorName = generateSnmpMonitorName(srcIp, srcPort); - AutoScaleVmGroupTO vmGroupTO = loadBalancerTO.getAutoScaleVmGroupTO(); - AutoScaleVmProfileTO profileTO = vmGroupTO.getProfile(); - List policies = vmGroupTO.getPolicies(); - int interval = vmGroupTO.getInterval(); - int snmpPort = profileTO.getSnmpPort(); - String snmpCommunity = profileTO.getSnmpCommunity(); - long cur_prirotiy = 1; - - try - { - // Set min and max autoscale members; - // add lb vserver lb http 10.102.31.100 80 -minAutoscaleMinMembers 3 -maxAutoscaleMembers 10 - int minAutoScaleMembers = vmGroupTO.getMinMembers(); - int maxAutoScaleMembers = vmGroupTO.getMaxMembers(); - lbvserver vserver = new lbvserver(); - try { - vserver.set_name(nsVirtualServerName); - vserver.set_minautoscalemembers(minAutoScaleMembers); - vserver.set_maxautoscalemembers(maxAutoScaleMembers); - vserver.update(_netscalerService, vserver); - } catch (Exception e) { - // Ignore Exception on cleanup - if(!isCleanUp) throw e; - } - - /* AutoScale Config */ - // Add AutoScale Profile - // add autoscale profile lb_asprofile CLOUDSTACK -url -http:// 10.102.31.34:8080/client/api- -apiKey abcdef - // -sharedSecret xyzabc - String apiKey = profileTO.getAutoScaleUserApiKey(); - String secretKey = profileTO.getAutoScaleUserSecretKey(); - String url = profileTO.getCloudStackApiUrl(); - - com.citrix.netscaler.nitro.resource.config.autoscale.autoscaleprofile autoscaleProfile = new com.citrix.netscaler.nitro.resource.config.autoscale.autoscaleprofile(); - try { - autoscaleProfile.set_name(profileName); - autoscaleProfile.set_type("CLOUDSTACK"); - autoscaleProfile.set_apikey(apiKey); - autoscaleProfile.set_sharedsecret(secretKey); - autoscaleProfile.set_url(url); - autoscaleProfile.add(_netscalerService, autoscaleProfile); - } catch (Exception e) { - // Ignore Exception on cleanup - if(!isCleanUp) throw e; - } - - // Add Timer - nstimer timer = new nstimer(); - try { - timer.set_name(timerName); - timer.set_interval(interval); - timer.add(_netscalerService, timer); - } catch (Exception e) { - // Ignore Exception on cleanup - if(!isCleanUp) throw e; - } - - // AutoScale Actions - Integer scaleUpQuietTime = null; - Integer scaleDownQuietTime = null; - for (AutoScalePolicyTO autoScalePolicyTO : policies) { - if(scaleUpQuietTime == null) { - if(isScaleUpPolicy(autoScalePolicyTO)) { - scaleUpQuietTime = autoScalePolicyTO.getQuietTime(); - if(scaleDownQuietTime != null) { - break; - } - } - } - if(scaleDownQuietTime == null) { - if(isScaleDownPolicy(autoScalePolicyTO)) { - scaleDownQuietTime = autoScalePolicyTO.getQuietTime(); - if(scaleUpQuietTime != null) { - break; - } - } - } - } - - // Add AutoScale ScaleUp action - // add autoscale action lb_scaleUpAction provision -vserver lb -profilename lb_asprofile -params - // -lbruleid=1234&command=deployvm&zoneid=10&templateid=5&serviceofferingid=3- -quiettime 300 - com.citrix.netscaler.nitro.resource.config.autoscale.autoscaleaction scaleUpAction = new com.citrix.netscaler.nitro.resource.config.autoscale.autoscaleaction(); - try { - scaleUpAction.set_name(scaleUpActionName); - scaleUpAction.set_type("SCALE_UP"); // TODO: will this be called provision? - scaleUpAction.set_vserver(nsVirtualServerName); // Actions Vserver, the one that is autoscaled, with CS - // now both are same. Not exposed in API. - scaleUpAction.set_profilename(profileName); - scaleUpAction.set_quiettime(scaleUpQuietTime); - String scaleUpParameters = "command=deployVirtualMachine" + "&" + - ApiConstants.ZONE_ID + "=" + profileTO.getZoneId()+ "&" + - ApiConstants.SERVICE_OFFERING_ID + "=" + profileTO.getServiceOfferingId()+ "&" + - ApiConstants.TEMPLATE_ID + "=" + profileTO.getTemplateId()+ "&" + - ((profileTO.getOtherDeployParams() == null)? "" : (profileTO.getOtherDeployParams() + "&")) + - "lbruleid=" + loadBalancerTO.getId(); - scaleUpAction.set_parameters(scaleUpParameters); - scaleUpAction.add(_netscalerService, scaleUpAction); - } catch (Exception e) { - // Ignore Exception on cleanup - if(!isCleanUp) throw e; - } - - com.citrix.netscaler.nitro.resource.config.autoscale.autoscaleaction scaleDownAction = new com.citrix.netscaler.nitro.resource.config.autoscale.autoscaleaction(); - Integer destroyVmGracePeriod = profileTO.getDestroyVmGraceperiod(); - try { - scaleDownAction.set_name(scaleDownActionName); - scaleDownAction.set_type("SCALE_DOWN"); // TODO: will this be called de-provision? - scaleDownAction.set_vserver(nsVirtualServerName); // TODO: no global option as of now through Nitro. - // Testing cannot be done. - scaleDownAction.set_profilename(profileName); - scaleDownAction.set_quiettime(scaleDownQuietTime); - String scaleDownParameters = "command=destroyVirtualMachine" + "&" + - "lbruleid=" + loadBalancerTO.getId(); - scaleDownAction.set_parameters(scaleDownParameters); - scaleDownAction.set_vmdestroygraceperiod(destroyVmGracePeriod); - scaleDownAction.add(_netscalerService, scaleDownAction); - } catch (Exception e) { - // Ignore Exception on cleanup - if(!isCleanUp) throw e; - } - - /* Create min member policy */ - String minMemberPolicyName = generateAutoScaleMinPolicyName(srcIp, srcPort); - String minMemberPolicyExp = "SYS.VSERVER(\"" + nsVirtualServerName + "\").ACTIVESERVICES.LT(SYS.VSERVER(\"" + nsVirtualServerName + "\").MINAUTOSCALEMEMBERS)"; - addAutoScalePolicy(timerName, minMemberPolicyName, cur_prirotiy++, minMemberPolicyExp, scaleUpActionName, - interval, interval, isCleanUp); - - /* Create max member policy */ - String maxMemberPolicyName = generateAutoScaleMaxPolicyName(srcIp, srcPort); - String maxMemberPolicyExp = "SYS.VSERVER(\"" + nsVirtualServerName + "\").ACTIVESERVICES.GT(SYS.VSERVER(\"" + nsVirtualServerName + "\").MAXAUTOSCALEMEMBERS)"; - addAutoScalePolicy(timerName, maxMemberPolicyName, cur_prirotiy++, maxMemberPolicyExp, scaleDownActionName, - interval, interval, isCleanUp); - - /* Create Counters */ - HashMap snmpMetrics = new HashMap(); - for (AutoScalePolicyTO autoScalePolicyTO : policies) { - List conditions = autoScalePolicyTO.getConditions(); - String policyExpression = ""; - int snmpCounterNumber = 0; - for (ConditionTO conditionTO : conditions) { - CounterTO counterTO = conditionTO.getCounter(); - String counterName = counterTO.getName(); - String operator = conditionTO.getRelationalOperator(); - long threshold = conditionTO.getThreshold(); - - StringBuilder conditionExpression = new StringBuilder(); - Formatter formatter = new Formatter(conditionExpression, Locale.US); - - if(counterTO.getSource().equals("snmp")) - { - counterName = generateSnmpMetricName(counterName); - if(snmpMetrics.size() == 0) { - // Create Metric Table - //add lb metricTable lb_metric_table - lbmetrictable metricTable = new lbmetrictable(); - try { - metricTable.set_metrictable(mtName); - metricTable.add(_netscalerService, metricTable); - } catch (Exception e) { - // Ignore Exception on cleanup - if(!isCleanUp) throw e; - } - - // Create Monitor - // add lb monitor lb_metric_table_mon LOAD -destPort 161 -snmpCommunity public -metricTable - // lb_metric_table -interval - lbmonitor monitor = new lbmonitor(); - try { - monitor.set_monitorname(monitorName); - monitor.set_type("LOAD"); - monitor.set_destport(snmpPort); - monitor.set_snmpcommunity(snmpCommunity); - monitor.set_metrictable(mtName); - monitor.set_interval((int)(interval * 0.8)); - monitor.add(_netscalerService, monitor); - } catch (Exception e) { - // Ignore Exception on cleanup - if(!isCleanUp) throw e; - } - - // Bind monitor to servicegroup. - // bind lb monitor lb_metric_table_mon lb_autoscaleGroup -passive - servicegroup_lbmonitor_binding servicegroup_monitor_binding = new servicegroup_lbmonitor_binding(); - try { - servicegroup_monitor_binding.set_servicegroupname(serviceGroupName); - servicegroup_monitor_binding.set_monitor_name(monitorName); - // metrics, basically use it for autoscaling purpose only. - servicegroup_monitor_binding.set_passive(true); - servicegroup_lbmonitor_binding.add(_netscalerService, servicegroup_monitor_binding); - } catch (Exception e) { - // Ignore Exception on cleanup - if(!isCleanUp) throw e; - } - } - - boolean newMetric = !snmpMetrics.containsKey(counterName); - if(newMetric) { - snmpMetrics.put(counterName, snmpCounterNumber++); - } - - if(newMetric) - { - // bind lb metricTable lb_metric_table mem 1.3.6.1.4.1.2021.11.9.0 - String counterOid = counterTO.getValue(); - lbmetrictable_metric_binding metrictable_metric_binding = new lbmetrictable_metric_binding(); - try { - metrictable_metric_binding.set_metrictable(mtName); - metrictable_metric_binding.set_metric(counterName); - metrictable_metric_binding.set_Snmpoid(counterOid); - metrictable_metric_binding.add(_netscalerService, metrictable_metric_binding); - } catch (Exception e) { - // Ignore Exception on cleanup - if(!isCleanUp) throw e; - } - - // bind lb monitor lb_metric_table_mon -metric cpu -metricThreshold 1 - // lbmonitor_lbmetrictable_binding monitor_metrictable_binding = new lbmonitor_lbmetrictable_binding(); - lbmonitor_metric_binding monitor_metric_binding = new lbmonitor_metric_binding();; - try { - monitor_metric_binding.set_monitorname(monitorName); - monitor_metric_binding.set_metric(counterName); - monitor_metric_binding.set_metricthreshold(1); - monitor_metric_binding.add(_netscalerService, monitor_metric_binding); - } catch (Exception e) { - // Ignore Exception on cleanup - if(!isCleanUp) throw e; - } - } - // SYS.VSERVER("abcd").SNMP_TABLE(0).AVERAGE_VALUE.GT(80) - int counterIndex = snmpMetrics.get(counterName); // TODO: temporary fix. later on counter name will be added as a param to SNMP_TABLE. - formatter.format("SYS.VSERVER(\"%s\").SNMP_TABLE(%d).AVERAGE_VALUE.%s(%d)",nsVirtualServerName, counterIndex, operator, threshold); - } - else if (counterTO.getSource().equals("netscaler")) - { - //SYS.VSERVER("abcd").RESPTIME.GT(10) - formatter.format("SYS.VSERVER(\"%s\").%s.%s(%d)",nsVirtualServerName, counterTO.getValue(), operator, threshold); - } - if(policyExpression.length() != 0) { - policyExpression += " && "; - } - policyExpression += conditionExpression; - } - policyExpression = "(" + policyExpression + ")"; - - String policyId = Long.toString(autoScalePolicyTO.getId()); - String policyName = generateAutoScalePolicyName(srcIp, srcPort, policyId); - String action = null; - if(isScaleUpPolicy(autoScalePolicyTO)) { - action = scaleUpActionName; - String scaleUpCondition = "SYS.VSERVER(\"" + nsVirtualServerName + "\").ACTIVESERVICES.LT(SYS.VSERVER(\"" + nsVirtualServerName + "\").MAXAUTOSCALEMEMBERS)"; - policyExpression = scaleUpCondition + " && " + policyExpression; - } else { - action = scaleDownActionName; - String scaleDownCondition = "SYS.VSERVER(\"" + nsVirtualServerName + "\").ACTIVESERVICES.GT(SYS.VSERVER(\"" + nsVirtualServerName + "\").MINAUTOSCALEMEMBERS)"; - policyExpression = scaleDownCondition + " && " + policyExpression; - } - - addAutoScalePolicy(timerName, policyName, cur_prirotiy++, policyExpression, action, - autoScalePolicyTO.getDuration(), interval, isCleanUp); - - } - } catch (Exception ex) { - if(!isCleanUp) { - // Normal course, exception has occurred - disableAutoScaleConfig(loadBalancerTO, true); - throw ex; - - } else { - // Programming error. Exception should never be thrown afterall. - throw ex; - } - } - - return true; - } - - - @SuppressWarnings("static-access") - private synchronized boolean disableAutoScaleConfig(LoadBalancerTO loadBalancerTO, boolean isCleanUp) throws Exception { - String srcIp = loadBalancerTO.getSrcIp(); - int srcPort = loadBalancerTO.getSrcPort(); - - String nsVirtualServerName = generateNSVirtualServerName(srcIp, srcPort); - String profileName = generateAutoScaleProfileName(srcIp, srcPort); - String timerName = generateAutoScaleTimerName(srcIp, srcPort); - String scaleDownActionName = generateAutoScaleScaleDownActionName(srcIp, srcPort); - String scaleUpActionName = generateAutoScaleScaleUpActionName(srcIp, srcPort); - String mtName = generateSnmpMetricTableName(srcIp, srcPort); - String monitorName = generateSnmpMonitorName(srcIp, srcPort); - String serviceGroupName = generateAutoScaleServiceGroupName(srcIp, srcPort); - AutoScaleVmGroupTO vmGroupTO = loadBalancerTO.getAutoScaleVmGroupTO(); - List policies = vmGroupTO.getPolicies(); - String minMemberPolicyName = generateAutoScaleMinPolicyName(srcIp, srcPort); - String maxMemberPolicyName = generateAutoScaleMaxPolicyName(srcIp, srcPort); - - try { - - /* Delete min/max member policies */ - - removeAutoScalePolicy(timerName, minMemberPolicyName, isCleanUp); - - removeAutoScalePolicy(timerName, maxMemberPolicyName, isCleanUp); - - boolean isSnmp = false; - /* Create Counters */ - for (AutoScalePolicyTO autoScalePolicyTO : policies) { - List conditions = autoScalePolicyTO.getConditions(); - for (ConditionTO conditionTO : conditions) { - CounterTO counterTO = conditionTO.getCounter(); - if(counterTO.getSource().equals("snmp")) { - isSnmp = true; - break; - } - } - String policyId = Long.toString(autoScalePolicyTO.getId()); - String policyName = generateAutoScalePolicyName(srcIp, srcPort,policyId); - - // Removing Timer policy - removeAutoScalePolicy(timerName, policyName, isCleanUp); - } - - /* Delete AutoScale Config */ - // Delete AutoScale ScaleDown action - com.citrix.netscaler.nitro.resource.config.autoscale.autoscaleaction scaleDownAction = new com.citrix.netscaler.nitro.resource.config.autoscale.autoscaleaction(); - try { - scaleDownAction.set_name(scaleDownActionName); - scaleDownAction.delete(_netscalerService, scaleDownAction); - } catch (Exception e) { - // Ignore Exception on cleanup - if(!isCleanUp) throw e; - } - - // Delete AutoScale ScaleUp action - com.citrix.netscaler.nitro.resource.config.autoscale.autoscaleaction scaleUpAction = new com.citrix.netscaler.nitro.resource.config.autoscale.autoscaleaction(); - try { - scaleUpAction.set_name(scaleUpActionName); - scaleUpAction.delete(_netscalerService, scaleUpAction); - } catch (Exception e) { - // Ignore Exception on cleanup - if(!isCleanUp) throw e; - } - - // Delete Timer - nstimer timer = new nstimer(); - try { - timer.set_name(timerName); - timer.delete(_netscalerService, timer); - } catch (Exception e) { - // Ignore Exception on cleanup - if(!isCleanUp) throw e; - } - - // Delete AutoScale Profile - com.citrix.netscaler.nitro.resource.config.autoscale.autoscaleprofile autoscaleProfile = new com.citrix.netscaler.nitro.resource.config.autoscale.autoscaleprofile(); - try { - autoscaleProfile.set_name(profileName); - autoscaleProfile.delete(_netscalerService, autoscaleProfile); - } catch (Exception e) { - // Ignore Exception on cleanup - if(!isCleanUp) throw e; - } - - if(isSnmp) { - servicegroup_lbmonitor_binding servicegroup_monitor_binding = new servicegroup_lbmonitor_binding(); - try { - servicegroup_monitor_binding.set_monitor_name(monitorName); - servicegroup_monitor_binding.set_servicegroupname(serviceGroupName); - servicegroup_lbmonitor_binding.delete(_netscalerService, servicegroup_monitor_binding); - } catch (Exception e) { - // Ignore Exception on cleanup - if(!isCleanUp) throw e; - } - - // Delete Monitor - // rm lb monitor lb_metric_table_mon - com.citrix.netscaler.nitro.resource.config.lb.lbmonitor monitor = new com.citrix.netscaler.nitro.resource.config.lb.lbmonitor(); - try { - monitor.set_monitorname(monitorName); - monitor.set_type("LOAD"); - monitor.delete(_netscalerService, monitor); - } catch (Exception e) { - // Ignore Exception on cleanup - if(!isCleanUp) throw e; - } - - // Delete Metric Table - com.citrix.netscaler.nitro.resource.config.lb.lbmetrictable metricTable = new com.citrix.netscaler.nitro.resource.config.lb.lbmetrictable(); - try { - metricTable.set_metrictable(mtName); - metricTable.delete(_netscalerService, metricTable); - } catch (Exception e) { - // Ignore Exception on cleanup - if(!isCleanUp) throw e; - } - } - } catch (Exception ex) { - if(!isCleanUp) { - // Normal course, exception has occurred - enableAutoScaleConfig(loadBalancerTO, true); - throw ex; - } else { - // Programming error - throw ex; - } - } - - return true; - } - - - private synchronized void addAutoScalePolicy(String timerName,String policyName, long priority, String policyExpression, String action, - int duration, int interval, boolean isCleanUp) throws Exception { - // Adding a autoscale policy - // add timer policy lb_policy_scaleUp_cpu_mem -rule - (SYS.CUR_VSERVER.METRIC_TABLE(cpu).AVG_VAL.GT(80)- - // -action lb_scaleUpAction - autoscalepolicy timerPolicy = new autoscalepolicy(); - try { - timerPolicy.set_name(policyName); - timerPolicy.set_action(action); - timerPolicy.set_rule(policyExpression); - timerPolicy.add(_netscalerService, timerPolicy); - } catch (Exception e) { - // Ignore Exception on cleanup - if(!isCleanUp) throw e; - } - - // bind timer policy - // For now it is bound globally. - // bind timer trigger lb_astimer -policyName lb_policy_scaleUp -vserver lb -priority 1 -samplesize 5 - // TODO: later bind to lbvserver. bind timer trigger lb_astimer -policyName lb_policy_scaleUp -vserver lb -priority 1 -samplesize 5 - // -thresholdsize 5 - nstimer_autoscalepolicy_binding timer_policy_binding = new nstimer_autoscalepolicy_binding(); - int sampleSize = duration/interval; - try { - timer_policy_binding.set_name(timerName); - timer_policy_binding.set_policyname(policyName); - // timer_policy_binding.set_global("DEFAULT"); // vserver name is present in the expression, this is default now - timer_policy_binding.set_samplesize(sampleSize); - timer_policy_binding.set_thresholdsize(sampleSize); // We are not exposing this parameter as of now. - // i.e. n(m) is not exposed to CS user. So thresholdSize == sampleSize - timer_policy_binding.set_priority(priority); - timer_policy_binding.add(_netscalerService, timer_policy_binding); - } catch (Exception e) { - // Ignore Exception on cleanup - if(!isCleanUp) throw e; - } - } - - private void removeAutoScalePolicy(String timerName, String policyName, boolean isCleanUp) throws Exception { - // unbind timer policy - // unbbind timer trigger lb_astimer -policyName lb_policy_scaleUp - nstimer_autoscalepolicy_binding timer_policy_binding = new nstimer_autoscalepolicy_binding(); - try { - timer_policy_binding.set_name(timerName); - timer_policy_binding.set_policyname(policyName); - // timer_policy_binding.set_global("DEFAULT"); // by default only global bank - timer_policy_binding.delete(_netscalerService, timer_policy_binding); - } catch (Exception e) { - // Ignore Exception on cleanup - if(!isCleanUp) throw e; - } - - // Removing Timer policy - // rm timer policy lb_policy_scaleUp_cpu_mem - autoscalepolicy timerPolicy = new autoscalepolicy(); - try { - timerPolicy.set_name(policyName); - timerPolicy.delete(_netscalerService, timerPolicy); - } catch (Exception e) { - // Ignore Exception on cleanup - if(!isCleanUp) throw e; - } - - } - - - private boolean isAutoScaleSupportedInNetScaler() throws ExecutionException { - autoscaleprofile autoscaleProfile = new autoscaleprofile(); - try { - autoscaleProfile.get(_netscalerService); - } catch (Exception ex) { - // Looks like autoscale is not supported in this netscaler. - // TODO: Config team has introduce a new command to check - // the list of entities supported in a NetScaler. Can use that - // once it is present in AutoScale branch. - s_logger.warn("AutoScale is not supported in NetScaler"); - return false; - } - return true; - } - - private boolean isScaleUpPolicy(AutoScalePolicyTO autoScalePolicyTO) { - return autoScalePolicyTO.getAction().equals("scaleup"); - } - - private boolean isScaleDownPolicy(AutoScalePolicyTO autoScalePolicyTO) { - return autoScalePolicyTO.getAction().equals("scaledown"); - } - - private void saveConfiguration() throws ExecutionException { - try { - apiCallResult = nsconfig.save(_netscalerService); - if (apiCallResult.errorcode != 0) { - throw new ExecutionException("Error occured while saving configuration changes to Netscaler device due to " + apiCallResult.message); - } - } catch (nitro_exception e) { - throw new ExecutionException("Failed to save configuration changes to Netscaler device due to " + e.getMessage()); - } catch (Exception e) { - throw new ExecutionException("Failed to save configuration changes to Netscaler device due to " + e.getMessage()); - } - } - - private ExternalNetworkResourceUsageAnswer getPublicIpBytesSentAndReceived(ExternalNetworkResourceUsageCommand cmd) throws ExecutionException { - ExternalNetworkResourceUsageAnswer answer = new ExternalNetworkResourceUsageAnswer(cmd); - - try { - lbvserver_stats[] stats = lbvserver_stats.get(_netscalerService); - - if (stats == null || stats.length == 0) { - return answer; - } - - for (lbvserver_stats stat_entry : stats) { - String lbvserverName = stat_entry.get_name(); - lbvserver vserver = lbvserver.get(_netscalerService, lbvserverName); - if(vserver != null){ - String lbVirtualServerIp = vserver.get_ipv46(); - - long[] bytesSentAndReceived = answer.ipBytes.get(lbVirtualServerIp); - if (bytesSentAndReceived == null) { - bytesSentAndReceived = new long[]{0, 0}; - } - bytesSentAndReceived[0] += stat_entry.get_totalrequestbytes(); - bytesSentAndReceived[1] += stat_entry.get_totalresponsebytes(); - - if (bytesSentAndReceived[0] >= 0 && bytesSentAndReceived[1] >= 0) { - answer.ipBytes.put(lbVirtualServerIp, bytesSentAndReceived); - } - } - } - } catch (Exception e) { - s_logger.error("Failed to get bytes sent and recived statistics due to " + e); - throw new ExecutionException(e.getMessage()); - } - - return answer; - } - - private Answer retry(Command cmd, int numRetries) { - int numRetriesRemaining = numRetries - 1; - s_logger.warn("Retrying " + cmd.getClass().getSimpleName() + ". Number of retries remaining: " + numRetriesRemaining); - return executeRequest(cmd, numRetriesRemaining); - } - - private boolean shouldRetry(int numRetries) { - try { - if (numRetries > 0) { - login(); - return true; - } - } catch (Exception e) { - s_logger.error("Failed to log in to Netscaler device at " + _ip + " due to " + e.getMessage()); - } - return false; - } - - private String generateInatRuleName(String srcIp, String dstIP) { - return genObjectName("Cloud-Inat", srcIp); - } - - private String generateNSVirtualServerName(String srcIp, long srcPort) { - return genObjectName("Cloud-VirtualServer", srcIp, srcPort); - } - - private String generateNSServerName(String serverIP) { - return genObjectName("Cloud-Server-", serverIP); - } - - private String generateNSServiceName(String ip, long port) { - return genObjectName("Cloud-Service", ip, port); - } - - private String generateAutoScaleServiceGroupName(String srcIp, long srcPort) { - return genObjectName("Cloud-AutoScaleServiceGroup", srcIp, srcPort); - } - - private String generateAutoScaleTimerName(String srcIp, long srcPort) { - return genObjectName("Cloud-AutoScale-Timer", srcIp, srcPort); - } - - private String generateAutoScaleProfileName(String srcIp, long srcPort) { - return genObjectName("Cloud-AutoScale-Profile", srcIp, srcPort); - } - - private String generateAutoScaleScaleUpActionName(String srcIp, long srcPort) { - return genObjectName("Cloud-AutoScale-ScaleUpAction", srcIp, srcPort); - } - - private String generateAutoScaleScaleDownActionName(String srcIp, long srcPort) { - return genObjectName("Cloud-AutoScale-ScaleDownAction", srcIp, srcPort); - } - - private String generateAutoScalePolicyName(String srcIp, long srcPort, String poilcyId) { - return genObjectName("Cloud-AutoScale-Policy", srcIp, srcPort, poilcyId); - } - - private String generateAutoScaleMinPolicyName(String srcIp, long srcPort) { - return genObjectName("Cloud-AutoScale-Policy-Min", srcIp, srcPort); - } - - private String generateAutoScaleMaxPolicyName(String srcIp, long srcPort) { - return genObjectName("Cloud-AutoScale-Policy-Max", srcIp, srcPort); - } - - private String generateSnmpMetricTableName(String srcIp, long srcPort) { - return genObjectName("Cloud-MTbl", srcIp, srcPort); - } - - private String generateSnmpMonitorName(String srcIp, long srcPort) { - return genObjectName("Cloud-Mon", srcIp, srcPort); - } - - private String generateSnmpMetricName(String counterName) { - return counterName.replace(' ', '_'); - } - - private String genObjectName(Object... args) { - String objectName = ""; - for (int i = 0; i < args.length; i++) { - objectName += args[i]; - if (i != args.length -1) { - objectName += _objectNamePathSep; - } - } - return objectName; - } - - @Override - public IAgentControl getAgentControl() { - return null; - } - - @Override - public PingCommand getCurrentStatus(long id) { - return new PingCommand(Host.Type.ExternalLoadBalancer, id); - } - - @Override - public Type getType() { - return Host.Type.ExternalLoadBalancer; - } - - @Override - public void setAgentControl(IAgentControl agentControl) { - return; - } - - @Override - public String getName() { - return _name; - } - - @Override - public boolean start() { - return true; - } - - @Override - public boolean stop() { - return true; - } - - @Override - public void disconnected() { - return; - } -} +// Copyright 2012 Citrix Systems, Inc. Licensed under the +// Apache License, Version 2.0 (the "License"); you may not use this +// file except in compliance with the License. Citrix Systems, Inc. +// reserves all rights not expressly granted by the License. +// You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Automatically generated by addcopyright.py at 04/03/2012 +package com.cloud.network.resource; + +import java.util.Formatter; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import javax.naming.ConfigurationException; + +import org.apache.log4j.Logger; + +import com.citrix.netscaler.nitro.exception.nitro_exception; +import com.citrix.netscaler.nitro.resource.base.base_response; +import com.citrix.netscaler.nitro.resource.config.autoscale.autoscalepolicy; +import com.citrix.netscaler.nitro.resource.config.autoscale.autoscaleprofile; +import com.citrix.netscaler.nitro.resource.config.basic.server_service_binding; +import com.citrix.netscaler.nitro.resource.config.basic.servicegroup; +import com.citrix.netscaler.nitro.resource.config.basic.servicegroup_lbmonitor_binding; +import com.citrix.netscaler.nitro.resource.config.lb.lbmetrictable; +import com.citrix.netscaler.nitro.resource.config.lb.lbmetrictable_metric_binding; +import com.citrix.netscaler.nitro.resource.config.lb.lbmonitor; +import com.citrix.netscaler.nitro.resource.config.lb.lbmonitor_metric_binding; +import com.citrix.netscaler.nitro.resource.config.lb.lbvserver; +import com.citrix.netscaler.nitro.resource.config.lb.lbvserver_service_binding; +import com.citrix.netscaler.nitro.resource.config.lb.lbvserver_servicegroup_binding; +import com.citrix.netscaler.nitro.resource.config.network.Interface; +import com.citrix.netscaler.nitro.resource.config.network.inat; +import com.citrix.netscaler.nitro.resource.config.network.vlan; +import com.citrix.netscaler.nitro.resource.config.network.vlan_interface_binding; +import com.citrix.netscaler.nitro.resource.config.network.vlan_nsip_binding; +import com.citrix.netscaler.nitro.resource.config.ns.nsconfig; +import com.citrix.netscaler.nitro.resource.config.ns.nshardware; +import com.citrix.netscaler.nitro.resource.config.ns.nsip; +import com.citrix.netscaler.nitro.resource.config.ns.nstimer; +import com.citrix.netscaler.nitro.resource.config.ns.nstimer_autoscalepolicy_binding; +import com.citrix.netscaler.nitro.resource.stat.lb.lbvserver_stats; +import com.citrix.netscaler.nitro.service.nitro_service; +import com.citrix.netscaler.nitro.util.filtervalue; +import com.citrix.sdx.nitro.resource.config.device_profile; +import com.citrix.sdx.nitro.resource.config.mps; +import com.citrix.sdx.nitro.resource.config.ns; +import com.citrix.sdx.nitro.resource.config.xen_vpx_image; +import com.cloud.agent.IAgentControl; +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.Command; +import com.cloud.agent.api.ExternalNetworkResourceUsageAnswer; +import com.cloud.agent.api.ExternalNetworkResourceUsageCommand; +import com.cloud.agent.api.MaintainAnswer; +import com.cloud.agent.api.MaintainCommand; +import com.cloud.agent.api.PingCommand; +import com.cloud.agent.api.ReadyAnswer; +import com.cloud.agent.api.ReadyCommand; +import com.cloud.agent.api.StartupCommand; +import com.cloud.agent.api.StartupExternalLoadBalancerCommand; +import com.cloud.agent.api.routing.CreateLoadBalancerApplianceCommand; +import com.cloud.agent.api.routing.DestroyLoadBalancerApplianceCommand; +import com.cloud.agent.api.routing.IpAssocAnswer; +import com.cloud.agent.api.routing.IpAssocCommand; +import com.cloud.agent.api.routing.LoadBalancerConfigCommand; +import com.cloud.agent.api.routing.SetStaticNatRulesAnswer; +import com.cloud.agent.api.routing.SetStaticNatRulesCommand; +import com.cloud.agent.api.to.IpAddressTO; +import com.cloud.agent.api.to.LoadBalancerTO; +import com.cloud.agent.api.to.LoadBalancerTO.AutoScalePolicyTO; +import com.cloud.agent.api.to.LoadBalancerTO.AutoScaleVmGroupTO; +import com.cloud.agent.api.to.LoadBalancerTO.AutoScaleVmProfileTO; +import com.cloud.agent.api.to.LoadBalancerTO.ConditionTO; +import com.cloud.agent.api.to.LoadBalancerTO.CounterTO; +import com.cloud.agent.api.to.LoadBalancerTO.DestinationTO; +import com.cloud.agent.api.to.LoadBalancerTO.StickinessPolicyTO; +import com.cloud.agent.api.to.StaticNatRuleTO; +import com.cloud.api.ApiConstants; +import com.cloud.host.Host; +import com.cloud.host.Host.Type; +import com.cloud.network.rules.LbStickinessMethod.StickinessMethodType; +import com.cloud.resource.ServerResource; +import com.cloud.serializer.GsonHelper; +import com.cloud.utils.NumbersUtil; +import com.cloud.utils.Pair; +import com.cloud.utils.exception.ExecutionException; +import com.cloud.utils.net.NetUtils; +import com.google.gson.Gson; + +class NitroError { + static final int NS_RESOURCE_EXISTS = 273; + static final int NS_RESOURCE_NOT_EXISTS = 258; + static final int NS_NO_SERIVCE = 344; + static final int NS_OPERATION_NOT_PERMITTED = 257; + static final int NS_INTERFACE_ALREADY_BOUND_TO_VLAN = 2080; +} + +public class NetscalerResource implements ServerResource { + + // deployment configuration + private String _name; + private String _zoneId; + private String _physicalNetworkId; + public String _ip; + public String _username; + public String _password; + private String _publicInterface; + private String _privateInterface; + private Integer _numRetries; + private String _guid; + private boolean _inline; + private boolean _isSdx; + private boolean _cloudManaged; + private String _deviceName; + private String _publicIP; + private String _publicIPNetmask; + private String _publicIPGateway; + private String _publicIPVlan; + + private static final Logger s_logger = Logger.getLogger(NetscalerResource.class); + protected Gson _gson; + private String _objectNamePathSep = "-"; + + // interface to interact with VPX and MPX devices + com.citrix.netscaler.nitro.service.nitro_service _netscalerService; + + // interface to interact with service VM of the SDX appliance + com.citrix.sdx.nitro.service.nitro_service _netscalerSdxService; + + Long _timeout = new Long(100000); + base_response apiCallResult; + + public NetscalerResource() { + _gson = GsonHelper.getGsonLogger(); + } + + @Override + public boolean configure(String name, Map params) throws ConfigurationException { + try { + _name = (String) params.get("name"); + if (_name == null) { + throw new ConfigurationException("Unable to find name in the configuration parameters"); + } + + _zoneId = (String) params.get("zoneId"); + if (_zoneId == null) { + throw new ConfigurationException("Unable to find zone Id in the configuration parameters"); + } + + _physicalNetworkId = (String) params.get("physicalNetworkId"); + if (_physicalNetworkId == null) { + throw new ConfigurationException("Unable to find physical network id in the configuration parameters"); + } + + _ip = (String) params.get("ip"); + if (_ip == null) { + throw new ConfigurationException("Unable to find IP address in the configuration parameters"); + } + + _username = (String) params.get("username"); + if (_username == null) { + throw new ConfigurationException("Unable to find username in the configuration parameters"); + } + + _password = (String) params.get("password"); + if (_password == null) { + throw new ConfigurationException("Unable to find password in the configuration parameters"); + } + + _publicInterface = (String) params.get("publicinterface"); + if (_publicInterface == null) { + throw new ConfigurationException("Unable to find public interface in the configuration parameters"); + } + + _privateInterface = (String) params.get("privateinterface"); + if (_privateInterface == null) { + throw new ConfigurationException("Unable to find private interface in the configuration parameters"); + } + + _numRetries = NumbersUtil.parseInt((String) params.get("numretries"), 2); + + _guid = (String) params.get("guid"); + if (_guid == null) { + throw new ConfigurationException("Unable to find the guid in the configuration parameters"); + } + + _deviceName = (String) params.get("deviceName"); + if (_deviceName == null) { + throw new ConfigurationException("Unable to find the device name in the configuration parameters"); + } + + _isSdx = _deviceName.equalsIgnoreCase("NetscalerSDXLoadBalancer"); + + _inline = Boolean.parseBoolean((String) params.get("inline")); + + if (((String) params.get("cloudmanaged")) != null) { + _cloudManaged = Boolean.parseBoolean((String) params.get("cloudmanaged")); + } + + // validate device configuration parameters + login(); + validateDeviceType(_deviceName); + validateInterfaces(_publicInterface, _privateInterface); + + // enable load balancing feature + enableLoadBalancingFeature(); + + // if the the device is cloud stack provisioned then make it part of the public network + if (_cloudManaged) { + _publicIP = (String) params.get("publicip"); + _publicIPGateway = (String) params.get("publicipgateway"); + _publicIPNetmask = (String) params.get("publicipnetmask"); + _publicIPVlan = (String) params.get("publicipvlan"); + if ("untagged".equalsIgnoreCase(_publicIPVlan)) { + // if public network is un-tagged just add subnet IP + addSubnetIP(_publicIP, _publicIPNetmask); + } else { + // if public network is tagged then add vlan and bind subnet IP to the vlan + addGuestVlanAndSubnet(Long.parseLong(_publicIPVlan), _publicIP, _publicIPNetmask, false); + } + } + + return true; + } catch (Exception e) { + throw new ConfigurationException(e.getMessage()); + } + } + + private void login() throws ExecutionException { + try { + if (!_isSdx) { + _netscalerService = new com.citrix.netscaler.nitro.service.nitro_service(_ip, "https"); + _netscalerService.set_credential(_username, _password); + _netscalerService.set_timeout(_timeout); + apiCallResult = _netscalerService.login(); + if (apiCallResult.errorcode != 0) { + throw new ExecutionException("Failed to log in to Netscaler device at " + _ip + " due to error " + apiCallResult.errorcode + " and message " + apiCallResult.message); + } + } else { + _netscalerSdxService = new com.citrix.sdx.nitro.service.nitro_service(_ip, "https"); + _netscalerSdxService.set_credential(_username, _password); + com.citrix.sdx.nitro.resource.base.login login = _netscalerSdxService.login(); + if (login == null) { + throw new ExecutionException("Failed to log in to Netscaler device at " + _ip + " due to error " + apiCallResult.errorcode + " and message " + apiCallResult.message); + } + } + } catch (nitro_exception e) { + throw new ExecutionException("Failed to log in to Netscaler device at " + _ip + " due to " + e.getMessage()); + } catch (Exception e) { + throw new ExecutionException("Failed to log in to Netscaler device at " + _ip + " due to " + e.getMessage()); + } + } + + private void enableLoadBalancingFeature() throws ExecutionException { + if (_isSdx) { + return; + } + try { + String[] features = _netscalerService.get_enabled_features(); + if (features != null) { + for (String feature : features) { + if (feature.equalsIgnoreCase("LB")) { + return; + } + } + } + + // enable load balancing on the device + String[] feature = new String[1]; + feature[0] = "LB"; + apiCallResult = _netscalerService.enable_features(feature); + if (apiCallResult.errorcode != 0) { + throw new ExecutionException("Enabling load balancing feature on the device failed."); + } + } catch (nitro_exception e) { + throw new ExecutionException("Enabling load balancing feature on the device failed due to " + e.getMessage()); + } catch (Exception e) { + throw new ExecutionException("Enabling load balancing feature on the device failed due to " + e.getMessage()); + } + } + + private void validateInterfaces(String publicInterface, String privateInterface) throws ExecutionException { + try { + if (!_isSdx && !_cloudManaged) { + Interface publicIf = Interface.get(_netscalerService, publicInterface); + Interface privateIf = Interface.get(_netscalerService, privateInterface); + if (publicIf != null || privateIf != null) { + return; + } else { + throw new ExecutionException("Invalid interface name specified for public/private interfaces."); + } + } + } catch (nitro_exception e) { + if (e.getErrorCode() == NitroError.NS_RESOURCE_NOT_EXISTS) { + throw new ExecutionException("Invalid interface name specified for public and private interfaces."); + } else { + throw new ExecutionException("Failed to verify public interface and private intefaces are valid due to " + e.getMessage()); + } + } catch (Exception e) { + throw new ExecutionException("Failed to verify public interface and private intefaces are valid due to " + e.getMessage()); + } + } + + private void validateDeviceType(String deviceType) throws ExecutionException { + try { + if (!_isSdx && !_cloudManaged) { + nshardware nsHw = com.citrix.netscaler.nitro.resource.config.ns.nshardware.get(_netscalerService); + if (nsHw == null) { + throw new ExecutionException("Failed to get the hardware description of the Netscaler device at " + _ip); + } else { + if ((_deviceName.equalsIgnoreCase("NetscalerMPXLoadBalancer") && nsHw.get_hwdescription().contains("MPX")) + || (_deviceName.equalsIgnoreCase("NetscalerVPXLoadBalancer") && nsHw.get_hwdescription().contains("NetScaler Virtual Appliance"))) { + return; + } + throw new ExecutionException("Netscalar device type specified does not match with the actuall device type."); + } + } else if (_isSdx) { + mps serviceVM = mps.get(_netscalerSdxService); + if (serviceVM != null) { + if (serviceVM.get_platform().contains("SDX") || serviceVM.get_product().contains("SDX")) { + return; + } else { + throw new ExecutionException("Netscalar device type specified does not match with the actuall device type."); + } + } else { + throw new ExecutionException("Failed to get the hardware details of the Netscaler device at " + _ip); + } + } + } catch (Exception e) { + throw new ExecutionException("Failed to verify device type specified when matching with actuall device type due to " + e.getMessage()); + } + } + + @Override + public StartupCommand[] initialize() { + StartupExternalLoadBalancerCommand cmd = new StartupExternalLoadBalancerCommand(); + cmd.setName(_name); + cmd.setDataCenter(_zoneId); + cmd.setPod(""); + cmd.setPrivateIpAddress(_ip); + cmd.setStorageIpAddress(""); + cmd.setVersion(""); + cmd.setGuid(_guid); + return new StartupCommand[] { cmd }; + } + + @Override + public Answer executeRequest(Command cmd) { + Answer ans = executeRequest(cmd, _numRetries); + return ans; + } + + private Answer executeRequest(Command cmd, int numRetries) { + if (cmd instanceof ReadyCommand) { + return execute((ReadyCommand) cmd); + } else if (cmd instanceof MaintainCommand) { + return execute((MaintainCommand) cmd); + } else if (cmd instanceof IpAssocCommand) { + return execute((IpAssocCommand) cmd, numRetries); + } else if (cmd instanceof LoadBalancerConfigCommand) { + return execute((LoadBalancerConfigCommand) cmd, numRetries); + } else if (cmd instanceof ExternalNetworkResourceUsageCommand) { + return execute((ExternalNetworkResourceUsageCommand) cmd, numRetries); + } else if (cmd instanceof CreateLoadBalancerApplianceCommand) { + return execute((CreateLoadBalancerApplianceCommand) cmd, numRetries); + } else if (cmd instanceof DestroyLoadBalancerApplianceCommand) { + return execute((DestroyLoadBalancerApplianceCommand) cmd, numRetries); + } else if (cmd instanceof SetStaticNatRulesCommand) { + return execute((SetStaticNatRulesCommand) cmd, numRetries); + } else { + return Answer.createUnsupportedCommandAnswer(cmd); + } + } + + private Answer execute(ReadyCommand cmd) { + return new ReadyAnswer(cmd); + } + + protected Answer execute(MaintainCommand cmd) { + return new MaintainAnswer(cmd); + } + + private synchronized Answer execute(IpAssocCommand cmd, int numRetries) { + if (_isSdx) { + return Answer.createUnsupportedCommandAnswer(cmd); + } + + String[] results = new String[cmd.getIpAddresses().length]; + int i = 0; + try { + IpAddressTO[] ips = cmd.getIpAddresses(); + for (IpAddressTO ip : ips) { + long guestVlanTag = Long.valueOf(ip.getVlanId()); + String vlanSelfIp = ip.getVlanGateway(); + String vlanNetmask = ip.getVlanNetmask(); + + if (ip.isAdd()) { + // Add a new guest VLAN and its subnet and bind it to private interface + addGuestVlanAndSubnet(guestVlanTag, vlanSelfIp, vlanNetmask, true); + } else { + // Check and delete guest VLAN with this tag, self IP, and netmask + deleteGuestVlan(guestVlanTag, vlanSelfIp, vlanNetmask); + } + + saveConfiguration(); + results[i++] = ip.getPublicIp() + " - success"; + String action = ip.isAdd() ? "associate" : "remove"; + if (s_logger.isDebugEnabled()) { + s_logger.debug("Netscaler load balancer " + _ip + " successfully executed IPAssocCommand to " + action + " IP " + ip); + } + } + } catch (ExecutionException e) { + s_logger.error("Netscaler loadbalancer " + _ip + " failed to execute IPAssocCommand due to " + e.getMessage()); + if (shouldRetry(numRetries)) { + return retry(cmd, numRetries); + } else { + results[i++] = IpAssocAnswer.errorResult; + } + } + + return new IpAssocAnswer(cmd, results); + } + + private synchronized Answer execute(LoadBalancerConfigCommand cmd, int numRetries) { + try { + if (_isSdx) { + return Answer.createUnsupportedCommandAnswer(cmd); + } + + LoadBalancerTO[] loadBalancers = cmd.getLoadBalancers(); + if (loadBalancers == null) { + return new Answer(cmd); + } + + for (LoadBalancerTO loadBalancer : loadBalancers) { + String srcIp = loadBalancer.getSrcIp(); + int srcPort = loadBalancer.getSrcPort(); + String lbProtocol = getNetScalerProtocol(loadBalancer); + String lbAlgorithm = loadBalancer.getAlgorithm(); + String nsVirtualServerName = generateNSVirtualServerName(srcIp, srcPort); + + if (loadBalancer.isAutoScaleVmGroupTO()) { + applyAutoScaleConfig(loadBalancer); + return new Answer(cmd); + } + + boolean destinationsToAdd = false; + for (DestinationTO destination : loadBalancer.getDestinations()) { + if (!destination.isRevoked()) { + destinationsToAdd = true; + break; + } + } + + if (!loadBalancer.isRevoked() && destinationsToAdd) { + + // create a load balancing virtual server + addLBVirtualServer(nsVirtualServerName, srcIp, srcPort, lbAlgorithm, lbProtocol, loadBalancer.getStickinessPolicies(), null); + if (s_logger.isDebugEnabled()) { + s_logger.debug("Created load balancing virtual server " + nsVirtualServerName + " on the Netscaler device"); + } + + for (DestinationTO destination : loadBalancer.getDestinations()) { + + String nsServerName = generateNSServerName(destination.getDestIp()); + String nsServiceName = generateNSServiceName(destination.getDestIp(), destination.getDestPort()); + + if (!destination.isRevoked()) { + // add a new destination to deployed load balancing rule + + // add a new server + if (!nsServerExists(nsServerName)) { + com.citrix.netscaler.nitro.resource.config.basic.server nsServer = new com.citrix.netscaler.nitro.resource.config.basic.server(); + nsServer.set_name(nsServerName); + nsServer.set_ipaddress(destination.getDestIp()); + apiCallResult = com.citrix.netscaler.nitro.resource.config.basic.server.add(_netscalerService, nsServer); + if ((apiCallResult.errorcode != 0) && (apiCallResult.errorcode != NitroError.NS_RESOURCE_EXISTS)) { + throw new ExecutionException("Failed to add server " + destination.getDestIp() + " due to" + apiCallResult.message); + } + } + + // create a new service using the server added + if (!nsServiceExists(nsServiceName)) { + com.citrix.netscaler.nitro.resource.config.basic.service newService = new com.citrix.netscaler.nitro.resource.config.basic.service(); + newService.set_name(nsServiceName); + newService.set_port(destination.getDestPort()); + newService.set_servername(nsServerName); + newService.set_state("ENABLED"); + newService.set_servicetype(lbProtocol); + + apiCallResult = com.citrix.netscaler.nitro.resource.config.basic.service.add(_netscalerService, newService); + if (apiCallResult.errorcode != 0) { + throw new ExecutionException("Failed to create service " + nsServiceName + " using server " + nsServerName + " due to" + apiCallResult.message); + } + } + + // bind service to load balancing virtual server + if (!nsServiceBindingExists(nsVirtualServerName, nsServiceName)) { + com.citrix.netscaler.nitro.resource.config.lb.lbvserver_service_binding svcBinding = new com.citrix.netscaler.nitro.resource.config.lb.lbvserver_service_binding(); + svcBinding.set_name(nsVirtualServerName); + svcBinding.set_servicename(nsServiceName); + apiCallResult = com.citrix.netscaler.nitro.resource.config.lb.lbvserver_service_binding.add(_netscalerService, svcBinding); + + if (apiCallResult.errorcode != 0) { + throw new ExecutionException("Failed to bind service: " + nsServiceName + " to the lb virtual server: " + nsVirtualServerName + " on Netscaler device"); + } + } + if (s_logger.isDebugEnabled()) { + s_logger.debug("Successfully added LB destination: " + destination.getDestIp() + ":" + destination.getDestPort() + " to load balancer " + srcIp + ":" + srcPort); + } + } else { + // remove a destination from the deployed load balancing rule + com.citrix.netscaler.nitro.resource.config.lb.lbvserver_service_binding[] serviceBindings = com.citrix.netscaler.nitro.resource.config.lb.lbvserver_service_binding.get(_netscalerService, + nsVirtualServerName); + if (serviceBindings != null) { + for (com.citrix.netscaler.nitro.resource.config.lb.lbvserver_service_binding binding : serviceBindings) { + if (nsServiceName.equalsIgnoreCase(binding.get_servicename())) { + // delete the binding + apiCallResult = com.citrix.netscaler.nitro.resource.config.lb.lbvserver_service_binding.delete(_netscalerService, binding); + if (apiCallResult.errorcode != 0) { + throw new ExecutionException("Failed to delete the binding between the virtual server: " + nsVirtualServerName + " and service:" + nsServiceName + " due to" + + apiCallResult.message); + } + + // check if service is bound to any other virtual server + if (!isServiceBoundToVirtualServer(nsServiceName)) { + // no lb virtual servers are bound to this service so delete it + apiCallResult = com.citrix.netscaler.nitro.resource.config.basic.service.delete(_netscalerService, nsServiceName); + if (apiCallResult.errorcode != 0) { + throw new ExecutionException("Failed to delete service: " + nsServiceName + " due to " + apiCallResult.message); + } + } + + // delete the server if there is no associated services + server_service_binding[] services = server_service_binding.get(_netscalerService, nsServerName); + if ((services == null) || (services.length == 0)) { + apiCallResult = com.citrix.netscaler.nitro.resource.config.basic.server.delete(_netscalerService, nsServerName); + if (apiCallResult.errorcode != 0) { + throw new ExecutionException("Failed to remove server:" + nsServerName + " due to " + apiCallResult.message); + } + } + } + } + } + } + } + } else { + // delete the implemented load balancing rule and its destinations + lbvserver lbserver = getVirtualServerIfExisits(nsVirtualServerName); + if (lbserver != null) { + // unbind the all services associated with this virtual server + com.citrix.netscaler.nitro.resource.config.lb.lbvserver_service_binding[] serviceBindings = com.citrix.netscaler.nitro.resource.config.lb.lbvserver_service_binding.get(_netscalerService, + nsVirtualServerName); + + if (serviceBindings != null) { + for (com.citrix.netscaler.nitro.resource.config.lb.lbvserver_service_binding binding : serviceBindings) { + String serviceName = binding.get_servicename(); + apiCallResult = com.citrix.netscaler.nitro.resource.config.lb.lbvserver_service_binding.delete(_netscalerService, binding); + if (apiCallResult.errorcode != 0) { + throw new ExecutionException("Failed to unbind service from the lb virtual server: " + nsVirtualServerName + " due to " + apiCallResult.message); + } + + com.citrix.netscaler.nitro.resource.config.basic.service svc = com.citrix.netscaler.nitro.resource.config.basic.service.get(_netscalerService, serviceName); + String nsServerName = svc.get_servername(); + + // check if service is bound to any other virtual server + if (!isServiceBoundToVirtualServer(serviceName)) { + // no lb virtual servers are bound to this service so delete it + apiCallResult = com.citrix.netscaler.nitro.resource.config.basic.service.delete(_netscalerService, serviceName); + if (apiCallResult.errorcode != 0) { + throw new ExecutionException("Failed to delete service: " + serviceName + " due to " + apiCallResult.message); + } + } + + // delete the server if no more services attached + server_service_binding[] services = server_service_binding.get(_netscalerService, nsServerName); + if ((services == null) || (services.length == 0)) { + apiCallResult = com.citrix.netscaler.nitro.resource.config.basic.server.delete(_netscalerService, nsServerName); + if (apiCallResult.errorcode != 0) { + throw new ExecutionException("Failed to remove server:" + nsServerName + " due to " + apiCallResult.message); + } + } + } + } + removeLBVirtualServer(nsVirtualServerName); + } + } + } + + if (s_logger.isInfoEnabled()) { + s_logger.info("Successfully executed resource LoadBalancerConfigCommand: " + _gson.toJson(cmd)); + } + + saveConfiguration(); + return new Answer(cmd); + } catch (ExecutionException e) { + s_logger.error("Failed to execute LoadBalancerConfigCommand due to ", e); + if (shouldRetry(numRetries)) { + return retry(cmd, numRetries); + } else { + return new Answer(cmd, e); + } + } catch (Exception e) { + s_logger.error("Failed to execute LoadBalancerConfigCommand due to ", e); + if (shouldRetry(numRetries)) { + return retry(cmd, numRetries); + } else { + return new Answer(cmd, e); + } + } + } + + private synchronized Answer execute(CreateLoadBalancerApplianceCommand cmd, int numRetries) { + + if (!_isSdx) { + return Answer.createUnsupportedCommandAnswer(cmd); + } + + try { + String vpxName = "Cloud-VPX-" + cmd.getLoadBalancerIP(); + String username = "admin"; + String password = "admin"; + + ns ns_obj = new ns(); + ns_obj.set_name(vpxName); + ns_obj.set_ip_address(cmd.getLoadBalancerIP()); + ns_obj.set_netmask(cmd.getNetmask()); + ns_obj.set_gateway(cmd.getGateway()); + ns_obj.set_username(username); + ns_obj.set_password(password); + + // configure VPX instances with defaults + ns_obj.set_feature_license("Standard"); + ns_obj.set_memory_total(new Double(2048)); + ns_obj.set_throughput(new Double(1000)); + ns_obj.set_pps(new Double(1000000)); + ns_obj.set_number_of_ssl_cores(0); + + // use the first device profile available on the SDX to create an instance of VPX + device_profile[] profiles = device_profile.get(_netscalerSdxService); + if (!(profiles != null && profiles.length >= 1)) { + new Answer(cmd, new ExecutionException("Failed to create VPX instance on the netscaler SDX device " + _ip + + " as there are no admin profile to use for creating VPX.")); + } + String profileName = profiles[0].get_name(); + ns_obj.set_nsroot_profile(profileName); + + // use the first VPX image of the available VPX images on the SDX to create an instance of VPX + // TODO: should enable the option to choose the template while adding the SDX device in to CloudStack + xen_vpx_image[] vpxImages = xen_vpx_image.get(_netscalerSdxService); + if (!(vpxImages != null && vpxImages.length >= 1)) { + new Answer(cmd, new ExecutionException("Failed to create VPX instance on the netscaler SDX device " + _ip + + " as there are no VPX images on SDX to use for creating VPX.")); + } + String imageName = vpxImages[0].get_file_name(); + ns_obj.set_image_name(imageName); + + String publicIf = _publicInterface; + String privateIf = _privateInterface; + + // enable only the interfaces that will be used by VPX + enableVPXInterfaces(_publicInterface, _privateInterface, ns_obj); + + // create new VPX instance + ns newVpx = ns.add(_netscalerSdxService, ns_obj); + + if (newVpx == null) { + new Answer(cmd, new ExecutionException("Failed to create VPX instance on the netscaler SDX device " + _ip)); + } + + // wait for VPX instance to start-up + long startTick = System.currentTimeMillis(); + long startWaitMilliSeconds = 600000; + while (!newVpx.get_ns_state().equalsIgnoreCase("up") && System.currentTimeMillis() - startTick < startWaitMilliSeconds) { + try { + Thread.sleep(10000); + } catch (InterruptedException e) { + } + ns refreshNsObj = new ns(); + refreshNsObj.set_id(newVpx.get_id()); + newVpx = ns.get(_netscalerSdxService, refreshNsObj); + } + + // if vpx instance never came up then error out + if (!newVpx.get_ns_state().equalsIgnoreCase("up")) { + return new Answer(cmd, new ExecutionException("Failed to start VPX instance " + vpxName + " created on the netscaler SDX device " + _ip)); + } + + // wait till NS service in side VPX is actually ready + startTick = System.currentTimeMillis(); + boolean nsServiceUp = false; + long nsServiceWaitMilliSeconds = 60000; + while (System.currentTimeMillis() - startTick < nsServiceWaitMilliSeconds) { + try { + nitro_service _netscalerService = new nitro_service(cmd.getLoadBalancerIP(), "https"); + _netscalerService.set_credential(username, password); + _netscalerService.set_timeout(_timeout); + apiCallResult = _netscalerService.login(); + if (apiCallResult.errorcode == 0) { + nsServiceUp = true; + break; + } + } catch (Exception e) { + Thread.sleep(10000); + continue; + } + } + + if (!nsServiceUp) { + return new Answer(cmd, new ExecutionException("Failed to create VPX instance " + vpxName + " on the netscaler SDX device " + _ip)); + } + + if (s_logger.isInfoEnabled()) { + s_logger.info("Successfully provisioned VPX instance " + vpxName + " on the Netscaler SDX device " + _ip); + } + + // physical interfaces on the SDX range from 10/1 to 10/8 & 1/1 to 1/8 of which two different port or same + // port can be used for public and private interfaces + // However the VPX instances created will have interface range start from 10/1 but will only have as many + // interfaces enabled while creating the VPX instance + // So due to this, we need to map public & private interface on SDX to correct public & private interface of + // VPX + + int publicIfnum = Integer.parseInt(_publicInterface.substring(_publicInterface.lastIndexOf("/") + 1)); + int privateIfnum = Integer.parseInt(_privateInterface.substring(_privateInterface.lastIndexOf("/") + 1)); + + if (_publicInterface.startsWith("10/") && _privateInterface.startsWith("10/")) { + if (publicIfnum == privateIfnum) { + publicIf = "10/1"; + privateIf = "10/1"; + } else if (publicIfnum > privateIfnum) { + privateIf = "10/1"; + publicIf = "10/2"; + } else { + publicIf = "10/1"; + privateIf = "10/2"; + } + } else if (_publicInterface.startsWith("1/") && _privateInterface.startsWith("1/")) { + if (publicIfnum == privateIfnum) { + publicIf = "1/1"; + privateIf = "1/1"; + } else if (publicIfnum > privateIfnum) { + privateIf = "1/1"; + publicIf = "1/2"; + } else { + publicIf = "1/1"; + privateIf = "1/2"; + } + } else if (_publicInterface.startsWith("1/") && _privateInterface.startsWith("10/")) { + publicIf = "1/1"; + privateIf = "10/1"; + } else if (_publicInterface.startsWith("10/") && _privateInterface.startsWith("1/")) { + publicIf = "10/1"; + privateIf = "1/1"; + } + + return new CreateLoadBalancerApplianceAnswer(cmd, true, "provisioned VPX instance", "NetscalerVPXLoadBalancer", "Netscaler", new NetscalerResource(), + publicIf, privateIf, _username, _password); + } catch (Exception e) { + if (shouldRetry(numRetries)) { + return retry(cmd, numRetries); + } + return new CreateLoadBalancerApplianceAnswer(cmd, false, "failed to provisioned VPX instance due to " + e.getMessage(), null, null, null, null, null, null, null); + } + } + + private void enableVPXInterfaces(String publicIf, String privateIf, ns ns_obj) { + // enable VPX to use 10 gigabit Ethernet interfaces if public/private interface + // on SDX is a 10Gig interface + if (publicIf.equals("10/1") || privateIf.equals("10/1")) { + ns_obj.set_if_10_1(new Boolean(true)); + } + + if (publicIf.equals("10/2") || privateIf.equals("10/2")) { + ns_obj.set_if_10_2(new Boolean(true)); + } + + if (publicIf.equals("10/3") || privateIf.equals("10/3")) { + ns_obj.set_if_10_3(new Boolean(true)); + } + + if (publicIf.equals("10/4") || privateIf.equals("10/4")) { + ns_obj.set_if_10_4(new Boolean(true)); + } + + if (publicIf.equals("10/5") || privateIf.equals("10/5")) { + ns_obj.set_if_10_5(new Boolean(true)); + } + + if (publicIf.equals("10/6") || privateIf.equals("10/6")) { + ns_obj.set_if_10_6(new Boolean(true)); + } + + if (publicIf.equals("10/7") || privateIf.equals("10/7")) { + ns_obj.set_if_10_7(new Boolean(true)); + } + + if (publicIf.equals("10/8") || privateIf.equals("10/8")) { + ns_obj.set_if_10_8(new Boolean(true)); + } + + // enable VPX to use 1 gigabit Ethernet interfaces if public/private interface + // on SDX is a 1Gig interface + if (publicIf.equals("1/1") || privateIf.equals("1/1")) { + ns_obj.set_if_1_1(new Boolean(true)); + } + + if (publicIf.equals("1/2") || privateIf.equals("1/2")) { + ns_obj.set_if_1_2(new Boolean(true)); + } + + if (publicIf.equals("1/3") || privateIf.equals("1/3")) { + ns_obj.set_if_1_3(new Boolean(true)); + } + + if (publicIf.equals("1/4") || privateIf.equals("1/4")) { + ns_obj.set_if_1_4(new Boolean(true)); + } + + if (publicIf.equals("1/5") || privateIf.equals("1/5")) { + ns_obj.set_if_1_5(new Boolean(true)); + } + + if (publicIf.equals("1/6") || privateIf.equals("1/6")) { + ns_obj.set_if_1_6(new Boolean(true)); + } + + if (publicIf.equals("1/7") || privateIf.equals("1/7")) { + ns_obj.set_if_1_7(new Boolean(true)); + } + + if (publicIf.equals("1/8") || privateIf.equals("1/8")) { + ns_obj.set_if_1_8(new Boolean(true)); + } + } + + private synchronized Answer execute(DestroyLoadBalancerApplianceCommand cmd, int numRetries) { + String vpxName = "Cloud-VPX-" + cmd.getLoadBalancerIP(); + if (!_isSdx) { + return Answer.createUnsupportedCommandAnswer(cmd); + } + + try { + ns vpxToDelete = null; + ns[] vpxInstances = ns.get(_netscalerSdxService); + for (ns vpx : vpxInstances) { + if (vpx.get_name().equals(vpxName)) { + vpxToDelete = vpx; + break; + } + } + + if (vpxToDelete == null) { + String msg = "There is no VPX instance " + vpxName + " on the Netscaler SDX device " + _ip + " to delete"; + s_logger.warn(msg); + return new DestroyLoadBalancerApplianceAnswer(cmd, true, msg); + } + + // destroy the VPX instance + ns nsDelObj = new ns(); + nsDelObj.set_id(vpxToDelete.get_id()); + vpxToDelete = ns.delete(_netscalerSdxService, nsDelObj); + String msg = "Deleted VPX instance " + vpxName + " on Netscaler SDX " + _ip + " successfully."; + s_logger.info(msg); + return new DestroyLoadBalancerApplianceAnswer(cmd, true, msg); + } catch (Exception e) { + if (shouldRetry(numRetries)) { + return retry(cmd, numRetries); + } + return new DestroyLoadBalancerApplianceAnswer(cmd, false, "Failed to delete VPX instance " + vpxName + " on Netscaler SDX " + _ip); + } + } + + private synchronized Answer execute(SetStaticNatRulesCommand cmd, int numRetries) { + + if (_isSdx) { + return Answer.createUnsupportedCommandAnswer(cmd); + } + + String[] results = new String[cmd.getRules().length]; + int i = 0; + boolean endResult = true; + + try { + for (StaticNatRuleTO rule : cmd.getRules()) { + String srcIp = rule.getSrcIp(); + String dstIP = rule.getDstIp(); + String iNatRuleName = generateInatRuleName(srcIp, dstIP); + inat iNatRule = null; + + if (!rule.revoked()) { + try { + iNatRule = inat.get(_netscalerService, iNatRuleName); + } catch (nitro_exception e) { + if (e.getErrorCode() != NitroError.NS_RESOURCE_NOT_EXISTS) { + throw e; + } + } + + if (iNatRule == null) { + iNatRule = new inat(); + iNatRule.set_name(iNatRuleName); + iNatRule.set_publicip(srcIp); + iNatRule.set_privateip(dstIP); + iNatRule.set_usnip("OFF"); + iNatRule.set_usip("ON"); + try { + apiCallResult = inat.add(_netscalerService, iNatRule); + } catch (nitro_exception e) { + if (e.getErrorCode() != NitroError.NS_RESOURCE_EXISTS) { + throw e; + } + } + s_logger.debug("Created Inat rule on the Netscaler device " + _ip + " to enable static NAT from " + srcIp + " to " + dstIP); + } + } else { + try { + inat.delete(_netscalerService, iNatRuleName); + } catch (nitro_exception e) { + if (e.getErrorCode() != NitroError.NS_RESOURCE_NOT_EXISTS) { + throw e; + } + } + s_logger.debug("Deleted Inat rule on the Netscaler device " + _ip + " to remove static NAT from " + srcIp + " to " + dstIP); + } + + saveConfiguration(); + results[i++] = "Static nat rule from " + srcIp + " to " + dstIP + " successfully " + (rule.revoked() ? " revoked." : " created."); + } + } catch (Exception e) { + if (shouldRetry(numRetries)) { + return retry(cmd, numRetries); + } + results[i++] = "Configuring static nat rule failed due to " + e.getMessage(); + endResult = false; + return new SetStaticNatRulesAnswer(cmd, results, endResult); + } + + return new SetStaticNatRulesAnswer(cmd, results, endResult); + } + + private synchronized Answer execute(ExternalNetworkResourceUsageCommand cmd, int numRetries) { + try { + if (!_isSdx) { + return getPublicIpBytesSentAndReceived(cmd); + } else { + return Answer.createUnsupportedCommandAnswer(cmd); + } + } catch (ExecutionException e) { + if (shouldRetry(numRetries)) { + return retry(cmd, numRetries); + } else { + return new ExternalNetworkResourceUsageAnswer(cmd, e); + } + } + } + + private void addSubnetIP(String snip, String netmask) throws ExecutionException { + try { + nsip selfIp = new nsip(); + selfIp.set_ipaddress(snip); + selfIp.set_netmask(netmask); + selfIp.set_type("SNIP"); + apiCallResult = nsip.add(_netscalerService, selfIp); + if (apiCallResult.errorcode != 0) { + throw new ExecutionException("Failed to add SNIP object on the Netscaler device due to " + apiCallResult.message); + } + } catch (nitro_exception e) { + throw new ExecutionException("Failed to add SNIP object on the Netscaler device due to " + e.getMessage()); + } catch (Exception e) { + throw new ExecutionException("Failed to add SNIP object on the Netscaler device due to " + e.getMessage()); + } + } + + private void addGuestVlanAndSubnet(long vlanTag, String vlanSelfIp, String vlanNetmask, boolean guestVlan) throws ExecutionException { + try { + // add vlan object for guest VLAN + if (!nsVlanExists(vlanTag)) { + try { + vlan vlanObj = new vlan(); + vlanObj.set_id(vlanTag); + apiCallResult = vlan.add(_netscalerService, vlanObj); + if (apiCallResult.errorcode != 0) { + throw new ExecutionException("Failed to add new vlan with tag:" + vlanTag + "on the NetScaler device due to " + apiCallResult.message); + } + } catch (nitro_exception e) { + throw new ExecutionException("Failed to add new vlan with tag:" + vlanTag + "on the NetScaler device due to " + e.getMessage()); + } + } + + // add subnet IP object for this guest network + if (!nsSnipExists(vlanSelfIp)) { + try { + nsip selfIp = new nsip(); + selfIp.set_ipaddress(vlanSelfIp); + selfIp.set_netmask(vlanNetmask); + selfIp.set_type("SNIP"); + apiCallResult = nsip.add(_netscalerService, selfIp); + if (apiCallResult.errorcode != 0) { + throw new ExecutionException("Failed to add SNIP object for the guest network on the Netscaler device due to " + apiCallResult.message); + } + } catch (nitro_exception e) { + throw new ExecutionException("Failed to add SNIP object for the guest network on the Netscaler device due to " + e.getMessage()); + } + } + + // bind the vlan object to subnet IP object + if (!nsVlanNsipBindingExists(vlanTag, vlanSelfIp)) { + try { + vlan_nsip_binding ipVlanBinding = new vlan_nsip_binding(); + ipVlanBinding.set_id(vlanTag); + ipVlanBinding.set_ipaddress(vlanSelfIp); + ipVlanBinding.set_netmask(vlanNetmask); + apiCallResult = vlan_nsip_binding.add(_netscalerService, ipVlanBinding); + if (apiCallResult.errorcode != 0) { + throw new ExecutionException("Failed to bind VLAN with tag:" + vlanTag + " to the subnet due to " + apiCallResult.message); + } + } catch (nitro_exception e) { + throw new ExecutionException("Failed to bind VLAN with tage:" + vlanTag + " to the subnet due to " + e.getMessage()); + } + } + + // bind vlan object to the private interface + try { + vlan_interface_binding vlanBinding = new vlan_interface_binding(); + if (guestVlan) { + vlanBinding.set_ifnum(_privateInterface); + } else { + vlanBinding.set_ifnum(_publicInterface); + } + vlanBinding.set_tagged(true); + vlanBinding.set_id(vlanTag); + apiCallResult = vlan_interface_binding.add(_netscalerService, vlanBinding); + if (apiCallResult.errorcode != 0) { + String vlanInterface = guestVlan ? _privateInterface : _publicInterface; + throw new ExecutionException("Failed to bind vlan with tag:" + vlanTag + " with the interface " + vlanInterface + " due to " + apiCallResult.message); + } + } catch (nitro_exception e) { + if (!(e.getErrorCode() == NitroError.NS_INTERFACE_ALREADY_BOUND_TO_VLAN)) { + throw new ExecutionException("Failed to bind VLAN " + vlanTag + " with interface on the Netscaler device due to " + e.getMessage()); + } + } + } catch (nitro_exception e) { + throw new ExecutionException("Failed to implement guest network on the Netscaler device due to " + e.getMessage()); + } catch (Exception e) { + throw new ExecutionException("Failed to implement guest network on the Netscaler device due to " + e.getMessage()); + } + } + + private void deleteGuestVlan(long vlanTag, String vlanSelfIp, String vlanNetmask) throws ExecutionException { + try { + + // Delete all servers and associated services from this guest VLAN + deleteServersInGuestVlan(vlanTag, vlanSelfIp, vlanNetmask); + + // unbind vlan to the private interface + try { + vlan_interface_binding vlanIfBinding = new vlan_interface_binding(); + vlanIfBinding.set_id(vlanTag); + vlanIfBinding.set_ifnum(_privateInterface); + vlanIfBinding.set_tagged(true); + apiCallResult = vlan_interface_binding.delete(_netscalerService, vlanIfBinding); + if (apiCallResult.errorcode != 0) { + throw new ExecutionException("Failed to unbind vlan:" + vlanTag + " with the private interface due to " + apiCallResult.message); + } + } catch (nitro_exception e) { + // if Vlan to interface binding does not exist then ignore the exception and proceed + if (!(e.getErrorCode() == NitroError.NS_RESOURCE_NOT_EXISTS)) { + throw new ExecutionException("Failed to unbind vlan from the interface while shutdown of guest network on the Netscaler device due to " + e.getMessage()); + } + } + + // unbind the vlan to subnet + try { + vlan_nsip_binding vlanSnipBinding = new vlan_nsip_binding(); + vlanSnipBinding.set_netmask(vlanNetmask); + vlanSnipBinding.set_ipaddress(vlanSelfIp); + vlanSnipBinding.set_id(vlanTag); + apiCallResult = vlan_nsip_binding.delete(_netscalerService, vlanSnipBinding); + if (apiCallResult.errorcode != 0) { + throw new ExecutionException("Failed to unbind vlan:" + vlanTag + " with the subnet due to " + apiCallResult.message); + } + } catch (nitro_exception e) { + // if Vlan to subnet binding does not exist then ignore the exception and proceed + if (!(e.getErrorCode() == NitroError.NS_RESOURCE_NOT_EXISTS)) { + throw new ExecutionException("Failed to unbind vlan:" + vlanTag + " with the subnet due to " + e.getMessage()); + } + } + + // remove subnet IP + try { + nsip subnetIp = nsip.get(_netscalerService, vlanSelfIp); + apiCallResult = nsip.delete(_netscalerService, subnetIp); + if (apiCallResult.errorcode != 0) { + throw new ExecutionException("Failed to remove subnet ip:" + vlanSelfIp + " from the NetScaler device due to" + apiCallResult.message); + } + } catch (nitro_exception e) { + // if subnet SNIP does not exist then ignore the exception and proceed + if (!(e.getErrorCode() == NitroError.NS_RESOURCE_NOT_EXISTS)) { + throw new ExecutionException("Failed to remove subnet ip:" + vlanSelfIp + " from the NetScaler device due to" + e.getMessage()); + } + } + + // remove the vlan from the NetScaler device + if (nsVlanExists(vlanTag)) { + // remove vlan + apiCallResult = com.citrix.netscaler.nitro.resource.config.network.vlan.delete(_netscalerService, vlanTag); + if (apiCallResult.errorcode != 0) { + throw new ExecutionException("Failed to remove vlan with tag:" + vlanTag + "due to" + apiCallResult.message); + } + } + } catch (nitro_exception e) { + throw new ExecutionException("Failed to delete guest vlan network on the Netscaler device due to " + e.getMessage()); + } catch (Exception e) { + throw new ExecutionException("Failed to delete guest vlan network on the Netscaler device due to " + e.getMessage()); + } + } + + private boolean nsVlanExists(long vlanTag) throws ExecutionException { + try { + if (vlan.get(_netscalerService, new Long(vlanTag)) != null) { + return true; + } else { + return false; + } + } catch (nitro_exception e) { + if (e.getErrorCode() == NitroError.NS_RESOURCE_NOT_EXISTS) { + return false; + } else { + throw new ExecutionException("Failed to verify VLAN exists on the NetScaler device due to " + e.getMessage()); + } + } catch (Exception e) { + throw new ExecutionException("Failed to verify VLAN exists on the NetScaler device due to " + e.getMessage()); + } + } + + private boolean nsSnipExists(String subnetIP) throws ExecutionException { + try { + nsip snip = nsip.get(_netscalerService, subnetIP); + return (snip != null); + } catch (nitro_exception e) { + if (e.getErrorCode() == NitroError.NS_RESOURCE_NOT_EXISTS) { + return false; + } else { + throw new ExecutionException("Failed to verify if SNIP exists on the NetScaler device due to " + e.getMessage()); + } + } catch (Exception e) { + throw new ExecutionException("Failed to verify if SNIP exists on the NetScaler device due to " + e.getMessage()); + } + } + + private boolean nsServerExists(String serverName) throws ExecutionException { + try { + if (com.citrix.netscaler.nitro.resource.config.basic.server.get(_netscalerService, serverName) != null) { + return true; + } else { + return false; + } + } catch (nitro_exception e) { + if (e.getErrorCode() == NitroError.NS_RESOURCE_NOT_EXISTS) { + return false; + } else { + throw new ExecutionException("Failed to verify Server " + serverName + " exists on the NetScaler device due to " + e.getMessage()); + } + } catch (Exception e) { + throw new ExecutionException("Failed to verify Server " + serverName + " exists on the NetScaler device due to " + e.getMessage()); + } + } + + private boolean nsVlanNsipBindingExists(long vlanTag, String vlanSelfIp) throws ExecutionException { + try { + vlan_nsip_binding[] vlanNsipBindings = vlan_nsip_binding.get(_netscalerService, vlanTag); + if (vlanNsipBindings != null && vlanNsipBindings[0] != null && vlanNsipBindings[0].get_ipaddress().equalsIgnoreCase(vlanSelfIp)) { + return true; + } else { + return false; + } + } catch (nitro_exception e) { + if (e.getErrorCode() == NitroError.NS_RESOURCE_NOT_EXISTS) { + return false; + } else { + throw new ExecutionException("Failed to verify Vlan " + vlanTag + " to SNIP " + vlanSelfIp + " binding exists due to " + e.getMessage()); + } + } catch (Exception e) { + throw new ExecutionException("Failed to verify Vlan " + vlanTag + " to SNIP " + vlanSelfIp + " binding exists due to " + e.getMessage()); + } + } + + private lbvserver getVirtualServerIfExisits(String lbVServerName) throws ExecutionException { + try { + return lbvserver.get(_netscalerService, lbVServerName); + } catch (nitro_exception e) { + if (e.getErrorCode() == NitroError.NS_RESOURCE_NOT_EXISTS) { + return null; + } else { + throw new ExecutionException(e.getMessage()); + } + } catch (Exception e) { + throw new ExecutionException(e.getMessage()); + } + } + + private boolean isServiceBoundToVirtualServer(String serviceName) throws ExecutionException { + try { + lbvserver[] lbservers = lbvserver.get(_netscalerService); + for (lbvserver vserver : lbservers) { + filtervalue[] filter = new filtervalue[1]; + filter[0] = new filtervalue("servicename", serviceName); + lbvserver_service_binding[] result = lbvserver_service_binding.get_filtered(_netscalerService, vserver.get_name(), filter); + if (result != null && result.length > 0) { + return true; + } + } + return false; + } catch (Exception e) { + throw new ExecutionException("Failed to verify service " + serviceName + " is bound to any virtual server due to " + e.getMessage()); + } + } + + private boolean nsServiceExists(String serviceName) throws ExecutionException { + try { + if (com.citrix.netscaler.nitro.resource.config.basic.service.get(_netscalerService, serviceName) != null) { + return true; + } else { + return false; + } + } catch (nitro_exception e) { + if (e.getErrorCode() == NitroError.NS_NO_SERIVCE) { + return false; + } else { + throw new ExecutionException("Failed to verify service " + serviceName + " exists due to " + e.getMessage()); + } + } catch (Exception e) { + throw new ExecutionException("Failed to verify service " + serviceName + " exists due to " + e.getMessage()); + } + } + + private boolean nsServiceBindingExists(String lbVirtualServer, String serviceName) throws ExecutionException { + try { + com.citrix.netscaler.nitro.resource.config.lb.lbvserver_service_binding[] serviceBindings = com.citrix.netscaler.nitro.resource.config.lb.lbvserver_service_binding.get(_netscalerService, lbVirtualServer); + if (serviceBindings != null) { + for (com.citrix.netscaler.nitro.resource.config.lb.lbvserver_service_binding binding : serviceBindings) { + if (serviceName.equalsIgnoreCase(binding.get_servicename())) { + return true; + } + } + } + return false; + } catch (nitro_exception e) { + throw new ExecutionException("Failed to verify lb vserver " + lbVirtualServer + "and service " + serviceName + " binding exists due to " + e.getMessage()); + } catch (Exception e) { + throw new ExecutionException("Failed to verify lb vserver " + lbVirtualServer + "and service " + serviceName + " binding exists due to " + e.getMessage()); + } + } + + private boolean isServiceGroupBoundToVirtualServer(String nsVirtualServerName, String serviceGroupName) throws ExecutionException { + + lbvserver_servicegroup_binding vserver_servicegroup_binding = new lbvserver_servicegroup_binding(); + + try { + lbvserver_servicegroup_binding[] result = vserver_servicegroup_binding.get_filtered(_netscalerService, nsVirtualServerName, "servicegroupname:" + serviceGroupName); + if (result != null && result.length > 0) { + return true; + } + } catch (Exception e) { + throw new ExecutionException("Failed to verify lb vserver " + nsVirtualServerName + "and servicegrop " + serviceGroupName + " binding exists due to " + e.getMessage()); + } + return false; + + } + + private servicegroup getServiceGroupIfExisits(String lbVServerName) throws ExecutionException { + try { + return servicegroup.get(_netscalerService, lbVServerName); + } catch (nitro_exception e) { + if (e.getErrorCode() == NitroError.NS_RESOURCE_NOT_EXISTS) { + return null; + } else { + throw new ExecutionException(e.getMessage()); + } + } catch (Exception e) { + throw new ExecutionException(e.getMessage()); + } + } + + private void deleteServersInGuestVlan(long vlanTag, String vlanSelfIp, String vlanNetmask) throws ExecutionException { + try { + com.citrix.netscaler.nitro.resource.config.basic.server[] serverList = com.citrix.netscaler.nitro.resource.config.basic.server.get(_netscalerService); + + if (serverList == null) { + return; + } + + // remove the server and services associated with guest vlan + for (com.citrix.netscaler.nitro.resource.config.basic.server server : serverList) { + // check if server belong to same subnet as one associated with vlan + if (NetUtils.sameSubnet(vlanSelfIp, server.get_ipaddress(), vlanNetmask)) { + // first remove services associated with this server + com.citrix.netscaler.nitro.resource.config.basic.service serveicesList[] = com.citrix.netscaler.nitro.resource.config.basic.service.get(_netscalerService); + if (serveicesList != null) { + for (com.citrix.netscaler.nitro.resource.config.basic.service svc : serveicesList) { + if (svc.get_servername().equals(server.get_ipaddress())) { + apiCallResult = com.citrix.netscaler.nitro.resource.config.basic.service.delete(_netscalerService, svc.get_name()); + if (apiCallResult.errorcode != 0) { + throw new ExecutionException("Failed to remove service:" + svc.get_name()); + } + } + } + } + // remove the server + apiCallResult = com.citrix.netscaler.nitro.resource.config.basic.server.delete(_netscalerService, server.get_name()); + if (apiCallResult.errorcode != 0) { + throw new ExecutionException("Failed to remove server:" + server.get_name()); + } + } + } + } catch (Exception e) { + throw new ExecutionException("Failed to delete server and services in the guest vlan:" + vlanTag + " on the Netscaler device due to: " + e.getMessage()); + } + } + + private String getNetScalerProtocol(LoadBalancerTO loadBalancer) throws ExecutionException { + String port = Integer.toString(loadBalancer.getSrcPort()); + String lbProtocol = loadBalancer.getProtocol(); + StickinessPolicyTO[] stickyPolicies = loadBalancer.getStickinessPolicies(); + String nsProtocol = "TCP"; + + if ((stickyPolicies != null) && (stickyPolicies.length > 0) && (stickyPolicies[0] != null)) { + StickinessPolicyTO stickinessPolicy = stickyPolicies[0]; + if (StickinessMethodType.LBCookieBased.getName().equalsIgnoreCase(stickinessPolicy.getMethodName()) || + (StickinessMethodType.AppCookieBased.getName().equalsIgnoreCase(stickinessPolicy.getMethodName()))) { + nsProtocol = "HTTP"; + return nsProtocol; + } + } + + if (port.equals(NetUtils.HTTP_PORT)) { + nsProtocol = "HTTP"; + } else if (NetUtils.TCP_PROTO.equalsIgnoreCase(lbProtocol)) { + nsProtocol = "TCP"; + } else if (NetUtils.UDP_PROTO.equalsIgnoreCase(lbProtocol)) { + nsProtocol = "UDP"; + } + + return nsProtocol; + } + + private void addLBVirtualServer(String virtualServerName, String publicIp, int publicPort, String lbAlgorithm, String protocol, StickinessPolicyTO[] stickyPolicies, AutoScaleVmGroupTO vmGroupTO) + throws ExecutionException { + try { + String lbMethod; + if ("roundrobin".equalsIgnoreCase(lbAlgorithm)) { + lbMethod = "ROUNDROBIN"; + } else if ("leastconn".equalsIgnoreCase(lbAlgorithm)) { + lbMethod = "LEASTCONNECTION"; + } else if ("source".equalsIgnoreCase(lbAlgorithm)) { + lbMethod = "SOURCEIPHASH"; + } else { + throw new ExecutionException("Got invalid load balancing algorithm: " + lbAlgorithm + " in the load balancing rule"); + } + + boolean vserverExisis = false; + lbvserver vserver = getVirtualServerIfExisits(virtualServerName); + if (vserver != null) { + if (!vserver.get_servicetype().equalsIgnoreCase(protocol)) { + throw new ExecutionException("Can not update virtual server:" + virtualServerName + " as current protocol:" + vserver.get_servicetype() + " of virtual server is different from the " + + " intended protocol:" + protocol); + } + vserverExisis = true; + } + // Use new vserver always for configuration + vserver = new lbvserver(); + vserver.set_name(virtualServerName); + vserver.set_ipv46(publicIp); + vserver.set_port(publicPort); + vserver.set_servicetype(protocol); + vserver.set_lbmethod(lbMethod); + + // netmask can only be set for source IP load balancer algorithm + if (!lbMethod.equalsIgnoreCase("SOURCEIPHASH")) { + vserver.set_netmask(null); + vserver.set_v6netmasklen(null); + } + + if ((stickyPolicies != null) && (stickyPolicies.length > 0) && (stickyPolicies[0] != null)) { + long timeout = 2;// netscaler default 2 min + String cookieName = null; + StickinessPolicyTO stickinessPolicy = stickyPolicies[0]; + + // get the session persistence parameters + List> paramsList = stickinessPolicy.getParams(); + for (Pair param : paramsList) { + if ("holdtime".equalsIgnoreCase(param.first())) { + timeout = Long.parseLong(param.second()); + } else if ("name".equalsIgnoreCase(param.first())) { + cookieName = param.second(); + } + } + + // configure virtual server based on the persistence method + if (StickinessMethodType.LBCookieBased.getName().equalsIgnoreCase(stickinessPolicy.getMethodName())) { + vserver.set_persistencetype("COOKIEINSERT"); + } else if (StickinessMethodType.SourceBased.getName().equalsIgnoreCase(stickinessPolicy.getMethodName())) { + vserver.set_persistencetype("SOURCEIP"); + } else if (StickinessMethodType.AppCookieBased.getName().equalsIgnoreCase(stickinessPolicy.getMethodName())) { + vserver.set_persistencetype("RULE"); + vserver.set_rule("HTTP.REQ.HEADER(\"COOKIE\").VALUE(0).typecast_nvlist_t('=',';').value(\"" + cookieName + "\")"); + vserver.set_resrule("HTTP.RES.HEADER(\"SET-COOKIE\").VALUE(0).typecast_nvlist_t('=',';').value(\"" + cookieName + "\")"); + } else { + throw new ExecutionException("Got invalid session persistence method: " + stickinessPolicy.getMethodName() + " in the load balancing rule"); + } + + // set session persistence timeout + vserver.set_timeout(timeout); + } else { + // delete the LB stickyness policy + vserver.set_persistencetype("NONE"); + } + + if (vserverExisis) { + apiCallResult = lbvserver.update(_netscalerService, vserver); + } else { + apiCallResult = lbvserver.add(_netscalerService, vserver); + } + if (apiCallResult.errorcode != 0) { + throw new ExecutionException("Failed to create new load balancing virtual server:" + virtualServerName + " due to " + apiCallResult.message); + } + + if (s_logger.isDebugEnabled()) { + s_logger.debug("Created load balancing virtual server " + virtualServerName + " on the Netscaler device"); + } + } catch (nitro_exception e) { + s_logger.warn("Failed to create virtual server ", e); + throw new ExecutionException("Failed to create new virtual server:" + virtualServerName + " due to " + e.getMessage()); + } catch (Exception e) { + throw new ExecutionException("Failed to create new virtual server:" + virtualServerName + " due to " + e.getMessage()); + } + } + + private void removeLBVirtualServer(String virtualServerName) throws ExecutionException { + try { + lbvserver vserver = lbvserver.get(_netscalerService, virtualServerName); + if (vserver == null) { + return; + } + apiCallResult = lbvserver.delete(_netscalerService, vserver); + if (apiCallResult.errorcode != 0) { + throw new ExecutionException("Failed to delete virtual server:" + virtualServerName + " due to " + apiCallResult.message); + } + } catch (nitro_exception e) { + if (e.getErrorCode() == NitroError.NS_RESOURCE_NOT_EXISTS) { + return; + } else { + throw new ExecutionException("Failed remove virtual server:" + virtualServerName + " due to " + e.getMessage()); + } + } catch (Exception e) { + throw new ExecutionException("Failed to remove virtual server:" + virtualServerName + " due to " + e.getMessage()); + } + } + + public synchronized void applyAutoScaleConfig(LoadBalancerTO loadBalancer) throws Exception, ExecutionException { + + AutoScaleVmGroupTO vmGroupTO = loadBalancer.getAutoScaleVmGroupTO(); + if (!isAutoScaleSupportedInNetScaler()) { + throw new ExecutionException("AutoScale not supported in this version of NetScaler"); + } + if (vmGroupTO.getState().equals("new")) { + assert !loadBalancer.isRevoked(); + createAutoScaleConfig(loadBalancer); + } + else if (loadBalancer.isRevoked() || vmGroupTO.getState().equals("revoke")) { + removeAutoScaleConfig(loadBalancer); + } + else if (vmGroupTO.getState().equals("enabled")) { + assert !loadBalancer.isRevoked(); + enableAutoScaleConfig(loadBalancer, false); + } + else if (vmGroupTO.getState().equals("disabled")) { + assert !loadBalancer.isRevoked(); + disableAutoScaleConfig(loadBalancer, false); + } else { + // /// This should never happen + throw new ExecutionException("Unknown vmGroup State :" + vmGroupTO.getState()); + } + } + + @SuppressWarnings("static-access") + private synchronized boolean createAutoScaleConfig(LoadBalancerTO loadBalancerTO) throws ExecutionException, Exception { + + String srcIp = loadBalancerTO.getSrcIp(); + int srcPort = loadBalancerTO.getSrcPort(); + String lbProtocol = getNetScalerProtocol(loadBalancerTO); + String lbAlgorithm = loadBalancerTO.getAlgorithm(); + String vmGroupIdentifier = generateAutoScaleVmGroupIdentifier(loadBalancerTO); + String nsVirtualServerName = generateNSVirtualServerName(srcIp, srcPort); + AutoScaleVmGroupTO vmGroupTO = loadBalancerTO.getAutoScaleVmGroupTO(); + if (s_logger.isDebugEnabled()) { + s_logger.debug("Created load balancing virtual server " + nsVirtualServerName + " on the Netscaler device"); + } + addLBVirtualServer(nsVirtualServerName, srcIp, srcPort, lbAlgorithm, lbProtocol, loadBalancerTO.getStickinessPolicies(), vmGroupTO); + + String serviceGroupName = generateAutoScaleServiceGroupName(vmGroupIdentifier); + servicegroup serviceGroup = getServiceGroupIfExisits(serviceGroupName); + if (serviceGroup == null) { + // add servicegroup lb_autoscaleGroup -autoscale POLICY -memberPort 80 + int memberPort = vmGroupTO.getMemberPort(); + try { + serviceGroup = new servicegroup(); + serviceGroup.set_servicegroupname(serviceGroupName); + serviceGroup.set_servicetype(lbProtocol); + serviceGroup.set_autoscale("POLICY"); + serviceGroup.set_memberport(memberPort); + serviceGroup.add(_netscalerService, serviceGroup); + } catch (Exception e) { + throw e; + } + } + + if (!isServiceGroupBoundToVirtualServer(nsVirtualServerName, serviceGroupName)) { + // Bind autoscale service group + // bind lb vserver lb lb_autoscaleGroup + lbvserver_servicegroup_binding vserver_servicegroup_binding = new lbvserver_servicegroup_binding(); + + try { + vserver_servicegroup_binding.set_name(nsVirtualServerName); + vserver_servicegroup_binding.set_servicegroupname(serviceGroupName); + vserver_servicegroup_binding.add(_netscalerService, vserver_servicegroup_binding); + } catch (Exception e) { + throw e; + } + } + + // Create the autoscale config + enableAutoScaleConfig(loadBalancerTO, false); + return true; + } + + @SuppressWarnings("static-access") + private synchronized boolean removeAutoScaleConfig(LoadBalancerTO loadBalancerTO) throws Exception, ExecutionException { + String srcIp = loadBalancerTO.getSrcIp(); + int srcPort = loadBalancerTO.getSrcPort(); + String vmGroupIdentifier = generateAutoScaleVmGroupIdentifier(loadBalancerTO); + + String nsVirtualServerName = generateNSVirtualServerName(srcIp, srcPort); + String serviceGroupName = generateAutoScaleServiceGroupName(vmGroupIdentifier); + + if (loadBalancerTO.getAutoScaleVmGroupTO().getState().equals("enabled")) { + disableAutoScaleConfig(loadBalancerTO, false); + } + + if (isServiceGroupBoundToVirtualServer(nsVirtualServerName, serviceGroupName)) { + // UnBind autoscale service group + // unbind lb vserver lb lb_autoscaleGroup + lbvserver_servicegroup_binding vserver_servicegroup_binding = new lbvserver_servicegroup_binding(); + try { + vserver_servicegroup_binding.set_name(nsVirtualServerName); + vserver_servicegroup_binding.set_servicegroupname(serviceGroupName); + vserver_servicegroup_binding.delete(_netscalerService, vserver_servicegroup_binding); + } catch (Exception e) { + throw e; + } + } + + if (getServiceGroupIfExisits(serviceGroupName) != null) { + // Remove autoscale service group + com.citrix.netscaler.nitro.resource.config.basic.servicegroup serviceGroup = new com.citrix.netscaler.nitro.resource.config.basic.servicegroup(); + try { + serviceGroup.set_servicegroupname(serviceGroupName); + serviceGroup.delete(_netscalerService, serviceGroup); + } catch (Exception e) { + throw e; + } + } + + removeLBVirtualServer(nsVirtualServerName); + + return true; + } + + @SuppressWarnings("static-access") + private synchronized boolean enableAutoScaleConfig(LoadBalancerTO loadBalancerTO, boolean isCleanUp) throws Exception { + String vmGroupIdentifier = generateAutoScaleVmGroupIdentifier(loadBalancerTO); + String srcIp = loadBalancerTO.getSrcIp(); + int srcPort = loadBalancerTO.getSrcPort(); + + String nsVirtualServerName = generateNSVirtualServerName(srcIp, srcPort); + String serviceGroupName = generateAutoScaleServiceGroupName(vmGroupIdentifier); + String profileName = generateAutoScaleProfileName(vmGroupIdentifier); + String timerName = generateAutoScaleTimerName(vmGroupIdentifier); + String scaleDownActionName = generateAutoScaleScaleDownActionName(vmGroupIdentifier); + String scaleUpActionName = generateAutoScaleScaleUpActionName(vmGroupIdentifier); + String mtName = generateSnmpMetricTableName(vmGroupIdentifier); + String monitorName = generateSnmpMonitorName(vmGroupIdentifier); + AutoScaleVmGroupTO vmGroupTO = loadBalancerTO.getAutoScaleVmGroupTO(); + AutoScaleVmProfileTO profileTO = vmGroupTO.getProfile(); + List policies = vmGroupTO.getPolicies(); + int interval = vmGroupTO.getInterval(); + int snmpPort = profileTO.getSnmpPort(); + String snmpCommunity = profileTO.getSnmpCommunity(); + long cur_prirotiy = 1; + + try + { + // Set min and max autoscale members; + // add lb vserver lb http 10.102.31.100 80 -minAutoscaleMinMembers 3 -maxAutoscaleMembers 10 + int minAutoScaleMembers = vmGroupTO.getMinMembers(); + int maxAutoScaleMembers = vmGroupTO.getMaxMembers(); + lbvserver vserver = new lbvserver(); + try { + vserver.set_name(nsVirtualServerName); + vserver.set_minautoscalemembers(minAutoScaleMembers); + vserver.set_maxautoscalemembers(maxAutoScaleMembers); + vserver.update(_netscalerService, vserver); + } catch (Exception e) { + // Ignore Exception on cleanup + if (!isCleanUp) + throw e; + } + + /* AutoScale Config */ + // Add AutoScale Profile + // add autoscale profile lb_asprofile CLOUDSTACK -url -http:// 10.102.31.34:8080/client/api- -apiKey abcdef + // -sharedSecret xyzabc + String apiKey = profileTO.getAutoScaleUserApiKey(); + String secretKey = profileTO.getAutoScaleUserSecretKey(); + String url = profileTO.getCloudStackApiUrl(); + + com.citrix.netscaler.nitro.resource.config.autoscale.autoscaleprofile autoscaleProfile = new com.citrix.netscaler.nitro.resource.config.autoscale.autoscaleprofile(); + try { + autoscaleProfile.set_name(profileName); + autoscaleProfile.set_type("CLOUDSTACK"); + autoscaleProfile.set_apikey(apiKey); + autoscaleProfile.set_sharedsecret(secretKey); + autoscaleProfile.set_url(url); + autoscaleProfile.add(_netscalerService, autoscaleProfile); + } catch (Exception e) { + // Ignore Exception on cleanup + if (!isCleanUp) + throw e; + } + + // Add Timer + nstimer timer = new nstimer(); + try { + timer.set_name(timerName); + timer.set_interval(interval); + timer.add(_netscalerService, timer); + } catch (Exception e) { + // Ignore Exception on cleanup + if (!isCleanUp) + throw e; + } + + // AutoScale Actions + Integer scaleUpQuietTime = null; + Integer scaleDownQuietTime = null; + for (AutoScalePolicyTO autoScalePolicyTO : policies) { + if (scaleUpQuietTime == null) { + if (isScaleUpPolicy(autoScalePolicyTO)) { + scaleUpQuietTime = autoScalePolicyTO.getQuietTime(); + if (scaleDownQuietTime != null) { + break; + } + } + } + if (scaleDownQuietTime == null) { + if (isScaleDownPolicy(autoScalePolicyTO)) { + scaleDownQuietTime = autoScalePolicyTO.getQuietTime(); + if (scaleUpQuietTime != null) { + break; + } + } + } + } + + // Add AutoScale ScaleUp action + // add autoscale action lb_scaleUpAction provision -vserver lb -profilename lb_asprofile -params + // -lbruleid=1234&command=deployvm&zoneid=10&templateid=5&serviceofferingid=3- -quiettime 300 + com.citrix.netscaler.nitro.resource.config.autoscale.autoscaleaction scaleUpAction = new com.citrix.netscaler.nitro.resource.config.autoscale.autoscaleaction(); + try { + scaleUpAction.set_name(scaleUpActionName); + scaleUpAction.set_type("SCALE_UP"); // TODO: will this be called provision? + scaleUpAction.set_vserver(nsVirtualServerName); // Actions Vserver, the one that is autoscaled, with CS + // now both are same. Not exposed in API. + scaleUpAction.set_profilename(profileName); + scaleUpAction.set_quiettime(scaleUpQuietTime); + String scaleUpParameters = "command=deployVirtualMachine" + "&" + + ApiConstants.ZONE_ID + "=" + profileTO.getZoneId() + "&" + + ApiConstants.SERVICE_OFFERING_ID + "=" + profileTO.getServiceOfferingId() + "&" + + ApiConstants.TEMPLATE_ID + "=" + profileTO.getTemplateId() + "&" + + ((profileTO.getOtherDeployParams() == null) ? "" : (profileTO.getOtherDeployParams() + "&")) + + "lbruleid=" + loadBalancerTO.getUuid(); + scaleUpAction.set_parameters(scaleUpParameters); + scaleUpAction.add(_netscalerService, scaleUpAction); + } catch (Exception e) { + // Ignore Exception on cleanup + if (!isCleanUp) + throw e; + } + + com.citrix.netscaler.nitro.resource.config.autoscale.autoscaleaction scaleDownAction = new com.citrix.netscaler.nitro.resource.config.autoscale.autoscaleaction(); + Integer destroyVmGracePeriod = profileTO.getDestroyVmGraceperiod(); + try { + scaleDownAction.set_name(scaleDownActionName); + scaleDownAction.set_type("SCALE_DOWN"); // TODO: will this be called de-provision? + scaleDownAction.set_vserver(nsVirtualServerName); // TODO: no global option as of now through Nitro. + // Testing cannot be done. + scaleDownAction.set_profilename(profileName); + scaleDownAction.set_quiettime(scaleDownQuietTime); + String scaleDownParameters = "command=destroyVirtualMachine" + "&" + + "lbruleid=" + loadBalancerTO.getUuid(); + scaleDownAction.set_parameters(scaleDownParameters); + scaleDownAction.set_vmdestroygraceperiod(destroyVmGracePeriod); + scaleDownAction.add(_netscalerService, scaleDownAction); + } catch (Exception e) { + // Ignore Exception on cleanup + if (!isCleanUp) + throw e; + } + + /* Create min member policy */ + String minMemberPolicyName = generateAutoScaleMinPolicyName(vmGroupIdentifier); + String minMemberPolicyExp = "SYS.VSERVER(\"" + nsVirtualServerName + "\").ACTIVESERVICES.LT(SYS.VSERVER(\"" + nsVirtualServerName + "\").MINAUTOSCALEMEMBERS)"; + addAutoScalePolicy(timerName, minMemberPolicyName, cur_prirotiy++, minMemberPolicyExp, scaleUpActionName, + interval, interval, isCleanUp); + + /* Create max member policy */ + String maxMemberPolicyName = generateAutoScaleMaxPolicyName(vmGroupIdentifier); + String maxMemberPolicyExp = "SYS.VSERVER(\"" + nsVirtualServerName + "\").ACTIVESERVICES.GT(SYS.VSERVER(\"" + nsVirtualServerName + "\").MAXAUTOSCALEMEMBERS)"; + addAutoScalePolicy(timerName, maxMemberPolicyName, cur_prirotiy++, maxMemberPolicyExp, scaleDownActionName, + interval, interval, isCleanUp); + + /* Create Counters */ + HashMap snmpMetrics = new HashMap(); + for (AutoScalePolicyTO autoScalePolicyTO : policies) { + List conditions = autoScalePolicyTO.getConditions(); + String policyExpression = ""; + int snmpCounterNumber = 0; + for (ConditionTO conditionTO : conditions) { + CounterTO counterTO = conditionTO.getCounter(); + String counterName = counterTO.getName(); + String operator = conditionTO.getRelationalOperator(); + long threshold = conditionTO.getThreshold(); + + StringBuilder conditionExpression = new StringBuilder(); + Formatter formatter = new Formatter(conditionExpression, Locale.US); + + if (counterTO.getSource().equals("snmp")) + { + counterName = generateSnmpMetricName(counterName); + if (snmpMetrics.size() == 0) { + // Create Metric Table + // add lb metricTable lb_metric_table + lbmetrictable metricTable = new lbmetrictable(); + try { + metricTable.set_metrictable(mtName); + metricTable.add(_netscalerService, metricTable); + } catch (Exception e) { + // Ignore Exception on cleanup + if (!isCleanUp) + throw e; + } + + // Create Monitor + // add lb monitor lb_metric_table_mon LOAD -destPort 161 -snmpCommunity public -metricTable + // lb_metric_table -interval + lbmonitor monitor = new lbmonitor(); + try { + monitor.set_monitorname(monitorName); + monitor.set_type("LOAD"); + monitor.set_destport(snmpPort); + monitor.set_snmpcommunity(snmpCommunity); + monitor.set_metrictable(mtName); + monitor.set_interval((int) (interval * 0.8)); + monitor.add(_netscalerService, monitor); + } catch (Exception e) { + // Ignore Exception on cleanup + if (!isCleanUp) + throw e; + } + + // Bind monitor to servicegroup. + // bind lb monitor lb_metric_table_mon lb_autoscaleGroup -passive + servicegroup_lbmonitor_binding servicegroup_monitor_binding = new servicegroup_lbmonitor_binding(); + try { + servicegroup_monitor_binding.set_servicegroupname(serviceGroupName); + servicegroup_monitor_binding.set_monitor_name(monitorName); + // metrics, basically use it for autoscaling purpose only. + servicegroup_monitor_binding.set_passive(true); + servicegroup_lbmonitor_binding.add(_netscalerService, servicegroup_monitor_binding); + } catch (Exception e) { + // Ignore Exception on cleanup + if (!isCleanUp) + throw e; + } + } + + boolean newMetric = !snmpMetrics.containsKey(counterName); + if (newMetric) { + snmpMetrics.put(counterName, snmpCounterNumber++); + } + + if (newMetric) + { + // bind lb metricTable lb_metric_table mem 1.3.6.1.4.1.2021.11.9.0 + String counterOid = counterTO.getValue(); + lbmetrictable_metric_binding metrictable_metric_binding = new lbmetrictable_metric_binding(); + try { + metrictable_metric_binding.set_metrictable(mtName); + metrictable_metric_binding.set_metric(counterName); + metrictable_metric_binding.set_Snmpoid(counterOid); + metrictable_metric_binding.add(_netscalerService, metrictable_metric_binding); + } catch (Exception e) { + // Ignore Exception on cleanup + if (!isCleanUp) + throw e; + } + + // bind lb monitor lb_metric_table_mon -metric cpu -metricThreshold 1 + // lbmonitor_lbmetrictable_binding monitor_metrictable_binding = new +// lbmonitor_lbmetrictable_binding(); + lbmonitor_metric_binding monitor_metric_binding = new lbmonitor_metric_binding(); + ; + try { + monitor_metric_binding.set_monitorname(monitorName); + monitor_metric_binding.set_metric(counterName); + monitor_metric_binding.set_metricthreshold(1); + monitor_metric_binding.add(_netscalerService, monitor_metric_binding); + } catch (Exception e) { + // Ignore Exception on cleanup + if (!isCleanUp) + throw e; + } + } + // SYS.VSERVER("abcd").SNMP_TABLE(0).AVERAGE_VALUE.GT(80) + int counterIndex = snmpMetrics.get(counterName); // TODO: temporary fix. later on counter name +// will be added as a param to SNMP_TABLE. + formatter.format("SYS.VSERVER(\"%s\").SNMP_TABLE(%d).AVERAGE_VALUE.%s(%d)", nsVirtualServerName, counterIndex, operator, threshold); + } + else if (counterTO.getSource().equals("netscaler")) + { + // SYS.VSERVER("abcd").RESPTIME.GT(10) + formatter.format("SYS.VSERVER(\"%s\").%s.%s(%d)", nsVirtualServerName, counterTO.getValue(), operator, threshold); + } + if (policyExpression.length() != 0) { + policyExpression += " && "; + } + policyExpression += conditionExpression; + } + policyExpression = "(" + policyExpression + ")"; + + String policyId = Long.toString(autoScalePolicyTO.getId()); + String policyName = generateAutoScalePolicyName(vmGroupIdentifier, policyId); + String action = null; + if (isScaleUpPolicy(autoScalePolicyTO)) { + action = scaleUpActionName; + String scaleUpCondition = "SYS.VSERVER(\"" + nsVirtualServerName + "\").ACTIVESERVICES.LT(SYS.VSERVER(\"" + nsVirtualServerName + "\").MAXAUTOSCALEMEMBERS)"; + policyExpression = scaleUpCondition + " && " + policyExpression; + } else { + action = scaleDownActionName; + String scaleDownCondition = "SYS.VSERVER(\"" + nsVirtualServerName + "\").ACTIVESERVICES.GT(SYS.VSERVER(\"" + nsVirtualServerName + "\").MINAUTOSCALEMEMBERS)"; + policyExpression = scaleDownCondition + " && " + policyExpression; + } + + addAutoScalePolicy(timerName, policyName, cur_prirotiy++, policyExpression, action, + autoScalePolicyTO.getDuration(), interval, isCleanUp); + + } + } catch (Exception ex) { + if (!isCleanUp) { + // Normal course, exception has occurred + disableAutoScaleConfig(loadBalancerTO, true); + throw ex; + + } else { + // Programming error. Exception should never be thrown afterall. + throw ex; + } + } + + return true; + } + + @SuppressWarnings("static-access") + private synchronized boolean disableAutoScaleConfig(LoadBalancerTO loadBalancerTO, boolean isCleanUp) throws Exception { + + String vmGroupIdentifier = generateAutoScaleVmGroupIdentifier(loadBalancerTO); + + String profileName = generateAutoScaleProfileName(vmGroupIdentifier); + String timerName = generateAutoScaleTimerName(vmGroupIdentifier); + String scaleDownActionName = generateAutoScaleScaleDownActionName(vmGroupIdentifier); + String scaleUpActionName = generateAutoScaleScaleUpActionName(vmGroupIdentifier); + String mtName = generateSnmpMetricTableName(vmGroupIdentifier); + String monitorName = generateSnmpMonitorName(vmGroupIdentifier); + String serviceGroupName = generateAutoScaleServiceGroupName(vmGroupIdentifier); + AutoScaleVmGroupTO vmGroupTO = loadBalancerTO.getAutoScaleVmGroupTO(); + List policies = vmGroupTO.getPolicies(); + String minMemberPolicyName = generateAutoScaleMinPolicyName(vmGroupIdentifier); + String maxMemberPolicyName = generateAutoScaleMaxPolicyName(vmGroupIdentifier); + + try { + + /* Delete min/max member policies */ + + removeAutoScalePolicy(timerName, minMemberPolicyName, isCleanUp); + + removeAutoScalePolicy(timerName, maxMemberPolicyName, isCleanUp); + + boolean isSnmp = false; + /* Create Counters */ + for (AutoScalePolicyTO autoScalePolicyTO : policies) { + List conditions = autoScalePolicyTO.getConditions(); + for (ConditionTO conditionTO : conditions) { + CounterTO counterTO = conditionTO.getCounter(); + if (counterTO.getSource().equals("snmp")) { + isSnmp = true; + break; + } + } + String policyId = Long.toString(autoScalePolicyTO.getId()); + String policyName = generateAutoScalePolicyName(vmGroupIdentifier, policyId); + + // Removing Timer policy + removeAutoScalePolicy(timerName, policyName, isCleanUp); + } + + /* Delete AutoScale Config */ + // Delete AutoScale ScaleDown action + com.citrix.netscaler.nitro.resource.config.autoscale.autoscaleaction scaleDownAction = new com.citrix.netscaler.nitro.resource.config.autoscale.autoscaleaction(); + try { + scaleDownAction.set_name(scaleDownActionName); + scaleDownAction.delete(_netscalerService, scaleDownAction); + } catch (Exception e) { + // Ignore Exception on cleanup + if (!isCleanUp) + throw e; + } + + // Delete AutoScale ScaleUp action + com.citrix.netscaler.nitro.resource.config.autoscale.autoscaleaction scaleUpAction = new com.citrix.netscaler.nitro.resource.config.autoscale.autoscaleaction(); + try { + scaleUpAction.set_name(scaleUpActionName); + scaleUpAction.delete(_netscalerService, scaleUpAction); + } catch (Exception e) { + // Ignore Exception on cleanup + if (!isCleanUp) + throw e; + } + + // Delete Timer + nstimer timer = new nstimer(); + try { + timer.set_name(timerName); + timer.delete(_netscalerService, timer); + } catch (Exception e) { + // Ignore Exception on cleanup + if (!isCleanUp) + throw e; + } + + // Delete AutoScale Profile + com.citrix.netscaler.nitro.resource.config.autoscale.autoscaleprofile autoscaleProfile = new com.citrix.netscaler.nitro.resource.config.autoscale.autoscaleprofile(); + try { + autoscaleProfile.set_name(profileName); + autoscaleProfile.delete(_netscalerService, autoscaleProfile); + } catch (Exception e) { + // Ignore Exception on cleanup + if (!isCleanUp) + throw e; + } + + if (isSnmp) { + servicegroup_lbmonitor_binding servicegroup_monitor_binding = new servicegroup_lbmonitor_binding(); + try { + servicegroup_monitor_binding.set_monitor_name(monitorName); + servicegroup_monitor_binding.set_servicegroupname(serviceGroupName); + servicegroup_lbmonitor_binding.delete(_netscalerService, servicegroup_monitor_binding); + } catch (Exception e) { + // Ignore Exception on cleanup + if (!isCleanUp) + throw e; + } + + // Delete Monitor + // rm lb monitor lb_metric_table_mon + com.citrix.netscaler.nitro.resource.config.lb.lbmonitor monitor = new com.citrix.netscaler.nitro.resource.config.lb.lbmonitor(); + try { + monitor.set_monitorname(monitorName); + monitor.set_type("LOAD"); + monitor.delete(_netscalerService, monitor); + } catch (Exception e) { + // Ignore Exception on cleanup + if (!isCleanUp) + throw e; + } + + // Delete Metric Table + com.citrix.netscaler.nitro.resource.config.lb.lbmetrictable metricTable = new com.citrix.netscaler.nitro.resource.config.lb.lbmetrictable(); + try { + metricTable.set_metrictable(mtName); + metricTable.delete(_netscalerService, metricTable); + } catch (Exception e) { + // Ignore Exception on cleanup + if (!isCleanUp) + throw e; + } + } + } catch (Exception ex) { + if (!isCleanUp) { + // Normal course, exception has occurred + enableAutoScaleConfig(loadBalancerTO, true); + throw ex; + } else { + // Programming error + throw ex; + } + } + + return true; + } + + private synchronized void addAutoScalePolicy(String timerName, String policyName, long priority, String policyExpression, String action, + int duration, int interval, boolean isCleanUp) throws Exception { + // Adding a autoscale policy + // add timer policy lb_policy_scaleUp_cpu_mem -rule - (SYS.CUR_VSERVER.METRIC_TABLE(cpu).AVG_VAL.GT(80)- + // -action lb_scaleUpAction + autoscalepolicy timerPolicy = new autoscalepolicy(); + try { + timerPolicy.set_name(policyName); + timerPolicy.set_action(action); + timerPolicy.set_rule(policyExpression); + timerPolicy.add(_netscalerService, timerPolicy); + } catch (Exception e) { + // Ignore Exception on cleanup + if (!isCleanUp) + throw e; + } + + // bind timer policy + // For now it is bound globally. + // bind timer trigger lb_astimer -policyName lb_policy_scaleUp -vserver lb -priority 1 -samplesize 5 + // TODO: later bind to lbvserver. bind timer trigger lb_astimer -policyName lb_policy_scaleUp -vserver lb +// -priority 1 -samplesize 5 + // -thresholdsize 5 + nstimer_autoscalepolicy_binding timer_policy_binding = new nstimer_autoscalepolicy_binding(); + int sampleSize = duration / interval; + try { + timer_policy_binding.set_name(timerName); + timer_policy_binding.set_policyname(policyName); + // timer_policy_binding.set_global("DEFAULT"); // vserver name is present in the expression, this is default +// now + timer_policy_binding.set_samplesize(sampleSize); + timer_policy_binding.set_thresholdsize(sampleSize); // We are not exposing this parameter as of now. + // i.e. n(m) is not exposed to CS user. So thresholdSize == sampleSize + timer_policy_binding.set_priority(priority); + timer_policy_binding.add(_netscalerService, timer_policy_binding); + } catch (Exception e) { + // Ignore Exception on cleanup + if (!isCleanUp) + throw e; + } + } + + private void removeAutoScalePolicy(String timerName, String policyName, boolean isCleanUp) throws Exception { + // unbind timer policy + // unbbind timer trigger lb_astimer -policyName lb_policy_scaleUp + nstimer_autoscalepolicy_binding timer_policy_binding = new nstimer_autoscalepolicy_binding(); + try { + timer_policy_binding.set_name(timerName); + timer_policy_binding.set_policyname(policyName); + // timer_policy_binding.set_global("DEFAULT"); // by default only global bank + timer_policy_binding.delete(_netscalerService, timer_policy_binding); + } catch (Exception e) { + // Ignore Exception on cleanup + if (!isCleanUp) + throw e; + } + + // Removing Timer policy + // rm timer policy lb_policy_scaleUp_cpu_mem + autoscalepolicy timerPolicy = new autoscalepolicy(); + try { + timerPolicy.set_name(policyName); + timerPolicy.delete(_netscalerService, timerPolicy); + } catch (Exception e) { + // Ignore Exception on cleanup + if (!isCleanUp) + throw e; + } + + } + + private boolean isAutoScaleSupportedInNetScaler() throws ExecutionException { + autoscaleprofile autoscaleProfile = new autoscaleprofile(); + try { + autoscaleProfile.get(_netscalerService); + } catch (Exception ex) { + // Looks like autoscale is not supported in this netscaler. + // TODO: Config team has introduce a new command to check + // the list of entities supported in a NetScaler. Can use that + // once it is present in AutoScale branch. + s_logger.warn("AutoScale is not supported in NetScaler"); + return false; + } + return true; + } + + private boolean isScaleUpPolicy(AutoScalePolicyTO autoScalePolicyTO) { + return autoScalePolicyTO.getAction().equals("scaleup"); + } + + private boolean isScaleDownPolicy(AutoScalePolicyTO autoScalePolicyTO) { + return autoScalePolicyTO.getAction().equals("scaledown"); + } + + private void saveConfiguration() throws ExecutionException { + try { + apiCallResult = nsconfig.save(_netscalerService); + if (apiCallResult.errorcode != 0) { + throw new ExecutionException("Error occured while saving configuration changes to Netscaler device due to " + apiCallResult.message); + } + } catch (nitro_exception e) { + throw new ExecutionException("Failed to save configuration changes to Netscaler device due to " + e.getMessage()); + } catch (Exception e) { + throw new ExecutionException("Failed to save configuration changes to Netscaler device due to " + e.getMessage()); + } + } + + private ExternalNetworkResourceUsageAnswer getPublicIpBytesSentAndReceived(ExternalNetworkResourceUsageCommand cmd) throws ExecutionException { + ExternalNetworkResourceUsageAnswer answer = new ExternalNetworkResourceUsageAnswer(cmd); + + try { + lbvserver_stats[] stats = lbvserver_stats.get(_netscalerService); + + if (stats == null || stats.length == 0) { + return answer; + } + + for (lbvserver_stats stat_entry : stats) { + String lbvserverName = stat_entry.get_name(); + lbvserver vserver = lbvserver.get(_netscalerService, lbvserverName); + if (vserver != null) { + String lbVirtualServerIp = vserver.get_ipv46(); + + long[] bytesSentAndReceived = answer.ipBytes.get(lbVirtualServerIp); + if (bytesSentAndReceived == null) { + bytesSentAndReceived = new long[] { 0, 0 }; + } + bytesSentAndReceived[0] += stat_entry.get_totalrequestbytes(); + bytesSentAndReceived[1] += stat_entry.get_totalresponsebytes(); + + if (bytesSentAndReceived[0] >= 0 && bytesSentAndReceived[1] >= 0) { + answer.ipBytes.put(lbVirtualServerIp, bytesSentAndReceived); + } + } + } + } catch (Exception e) { + s_logger.error("Failed to get bytes sent and recived statistics due to " + e); + throw new ExecutionException(e.getMessage()); + } + + return answer; + } + + private Answer retry(Command cmd, int numRetries) { + int numRetriesRemaining = numRetries - 1; + s_logger.warn("Retrying " + cmd.getClass().getSimpleName() + ". Number of retries remaining: " + numRetriesRemaining); + return executeRequest(cmd, numRetriesRemaining); + } + + private boolean shouldRetry(int numRetries) { + try { + if (numRetries > 0) { + login(); + return true; + } + } catch (Exception e) { + s_logger.error("Failed to log in to Netscaler device at " + _ip + " due to " + e.getMessage()); + } + return false; + } + + private String generateInatRuleName(String srcIp, String dstIP) { + return genObjectName("Cloud-Inat", srcIp); + } + + private String generateNSVirtualServerName(String srcIp, long srcPort) { + return genObjectName("Cloud-VirtualServer", srcIp, srcPort); + } + + private String generateNSServerName(String serverIP) { + return genObjectName("Cloud-Server-", serverIP); + } + + private String generateNSServiceName(String ip, long port) { + return genObjectName("Cloud-Service", ip, port); + } + + private String generateAutoScaleVmGroupIdentifier(LoadBalancerTO lbTO) { + return lbTO.getSrcIp() + "-" + lbTO.getSrcPort(); + } + + private String generateAutoScaleServiceGroupName(String vmGroupIdentifier) { + return genObjectName("Cloud-AutoScale-SvcGrp", vmGroupIdentifier); + } + + private String generateAutoScaleTimerName(String vmGroupIdentifier) { + return genObjectName("Cloud-AutoScale-Timer", vmGroupIdentifier); + } + + private String generateAutoScaleProfileName(String vmGroupIdentifier) { + return genObjectName("Cloud-AutoScale-Profile", vmGroupIdentifier); + } + + private String generateAutoScaleScaleUpActionName(String vmGroupIdentifier) { + return genObjectName("Cloud-AutoScale-ScaleUpAction", vmGroupIdentifier); + } + + private String generateAutoScaleScaleDownActionName(String vmGroupIdentifier) { + return genObjectName("Cloud-AutoScale-ScaleDownAction", vmGroupIdentifier); + } + + private String generateAutoScalePolicyName(String vmGroupIdentifier, String poilcyId) { + return genObjectName("Cloud-AutoScale-Policy", vmGroupIdentifier, poilcyId); + } + + private String generateAutoScaleMinPolicyName(String vmGroupIdentifier) { + return genObjectName("Cloud-AutoScale-Policy-Min", vmGroupIdentifier); + } + + private String generateAutoScaleMaxPolicyName(String vmGroupIdentifier) { + return genObjectName("Cloud-AutoScale-Policy-Max", vmGroupIdentifier); + } + + private String generateSnmpMetricTableName(String vmGroupIdentifier) { + return genObjectName("Cloud-MTbl", vmGroupIdentifier); + } + + private String generateSnmpMonitorName(String vmGroupIdentifier) { + return genObjectName("Cloud-Mon", vmGroupIdentifier); + } + + private String generateSnmpMetricName(String counterName) { + return counterName.replace(' ', '_'); + } + + private String genObjectName(Object... args) { + String objectName = ""; + for (int i = 0; i < args.length; i++) { + objectName += args[i]; + if (i != args.length - 1) { + objectName += _objectNamePathSep; + } + } + return objectName; + } + + @Override + public IAgentControl getAgentControl() { + return null; + } + + @Override + public PingCommand getCurrentStatus(long id) { + return new PingCommand(Host.Type.ExternalLoadBalancer, id); + } + + @Override + public Type getType() { + return Host.Type.ExternalLoadBalancer; + } + + @Override + public void setAgentControl(IAgentControl agentControl) { + return; + } + + @Override + public String getName() { + return _name; + } + + @Override + public boolean start() { + return true; + } + + @Override + public boolean stop() { + return true; + } + + @Override + public void disconnected() { + return; + } +} diff --git a/server/src/com/cloud/network/ExternalLoadBalancerDeviceManagerImpl.java b/server/src/com/cloud/network/ExternalLoadBalancerDeviceManagerImpl.java index ef35b8b244b..fc0d56819d9 100644 --- a/server/src/com/cloud/network/ExternalLoadBalancerDeviceManagerImpl.java +++ b/server/src/com/cloud/network/ExternalLoadBalancerDeviceManagerImpl.java @@ -111,6 +111,7 @@ import com.cloud.vm.Nic.State; import com.cloud.vm.NicVO; import com.cloud.vm.dao.DomainRouterDao; import com.cloud.vm.dao.NicDao; + public abstract class ExternalLoadBalancerDeviceManagerImpl extends AdapterBase implements ExternalLoadBalancerDeviceManager, ResourceStateAdapter { @Inject @@ -148,6 +149,8 @@ public abstract class ExternalLoadBalancerDeviceManagerImpl extends AdapterBase @Inject UserStatisticsDao _userStatsDao; @Inject + LoadBalancerDao _lbDao; + @Inject NetworkDao _networkDao; @Inject DomainRouterDao _routerDao; @@ -795,7 +798,7 @@ public abstract class ExternalLoadBalancerDeviceManagerImpl extends AdapterBase boolean revoked = (rule.getState().equals(FirewallRule.State.Revoke)); String protocol = rule.getProtocol(); String algorithm = rule.getAlgorithm(); - Long lbId = rule.getId(); + String uuid = rule.getUuid(); String srcIp = _networkMgr.getIp(rule.getSourceIpAddressId()).getAddress().addr(); int srcPort = rule.getSourcePortStart(); List destinations = rule.getDestinations(); @@ -862,8 +865,8 @@ public abstract class ExternalLoadBalancerDeviceManagerImpl extends AdapterBase } if ((destinations != null && !destinations.isEmpty()) || rule.isAutoScaleConfig()) { - LoadBalancerTO loadBalancer = new LoadBalancerTO(lbId, srcIp, srcPort, protocol, algorithm, revoked, false, destinations, rule.getStickinessPolicies()); - if(rule.isAutoScaleConfig()) + LoadBalancerTO loadBalancer = new LoadBalancerTO(uuid, srcIp, srcPort, protocol, algorithm, revoked, false, destinations, rule.getStickinessPolicies()); + if (rule.isAutoScaleConfig()) loadBalancer.setAutoScaleVmGroup(rule.getAutoScaleVmGroup()); loadBalancersToApply.add(loadBalancer); } diff --git a/server/src/com/cloud/network/as/AutoScaleVmGroupVO.java b/server/src/com/cloud/network/as/AutoScaleVmGroupVO.java index cbeb5998134..ee99db0a0ce 100644 --- a/server/src/com/cloud/network/as/AutoScaleVmGroupVO.java +++ b/server/src/com/cloud/network/as/AutoScaleVmGroupVO.java @@ -178,4 +178,9 @@ public class AutoScaleVmGroupVO implements AutoScaleVmGroup { public void setLoadBalancerId(Long loadBalancerId) { this.loadBalancerId = loadBalancerId; } + + @Override + public String getUuid() { + return uuid; + } } diff --git a/server/src/com/cloud/network/element/NetscalerElement.java b/server/src/com/cloud/network/element/NetscalerElement.java index 6f6f17c6c46..165faa8d177 100644 --- a/server/src/com/cloud/network/element/NetscalerElement.java +++ b/server/src/com/cloud/network/element/NetscalerElement.java @@ -99,7 +99,7 @@ import com.google.gson.Gson; @Local(value = NetworkElement.class) public class NetscalerElement extends ExternalLoadBalancerDeviceManagerImpl implements LoadBalancingServiceProvider, NetscalerLoadBalancerElementService, ExternalLoadBalancerDeviceManager, IpDeployer, -StaticNatServiceProvider { + StaticNatServiceProvider { private static final Logger s_logger = Logger.getLogger(NetscalerElement.class); @@ -150,7 +150,7 @@ StaticNatServiceProvider { @Override public boolean implement(Network guestConfig, NetworkOffering offering, DeployDestination dest, ReservationContext context) throws ResourceUnavailableException, ConcurrentOperationException, - InsufficientNetworkCapacityException { + InsufficientNetworkCapacityException { if (!canHandle(guestConfig, Service.Lb)) { return false; @@ -163,7 +163,7 @@ StaticNatServiceProvider { } if (isBasicZoneNetwok(guestConfig)) { - // in basic zone there is nothing to be implemented/shutdown on the NetScaler appliance + // in basic zone there is nothing to be implemented/shutdown on the NetScaler appliance return true; } @@ -176,7 +176,7 @@ StaticNatServiceProvider { @Override public boolean prepare(Network config, NicProfile nic, VirtualMachineProfile vm, DeployDestination dest, ReservationContext context) throws ConcurrentOperationException, - InsufficientNetworkCapacityException, ResourceUnavailableException { + InsufficientNetworkCapacityException, ResourceUnavailableException { return true; } @@ -192,9 +192,9 @@ StaticNatServiceProvider { } if (isBasicZoneNetwok(guestConfig)) { - // in basic zone there is nothing to be implemented/shutdown on the NetScaler appliance + // in basic zone there is nothing to be implemented/shutdown on the NetScaler appliance return true; - } + } try { return manageGuestNetworkWithExternalLoadBalancer(false, guestConfig); @@ -523,7 +523,7 @@ StaticNatServiceProvider { @Override public boolean shutdownProviderInstances(PhysicalNetworkServiceProvider provider, ReservationContext context) throws ConcurrentOperationException, - ResourceUnavailableException { + ResourceUnavailableException { // TODO reset the configuration on all of the netscaler devices in this physical network return true; } @@ -544,17 +544,17 @@ StaticNatServiceProvider { } @Override - public boolean verifyServicesCombination(Set services) { + public boolean verifyServicesCombination(Set services) { Set netscalerServices = new HashSet(); netscalerServices.add(Service.Lb); netscalerServices.add(Service.StaticNat); // NetScaler can only act as Lb and Static Nat service provider if (services != null && !services.isEmpty() && !netscalerServices.containsAll(services)) { - String servicesList = ""; - for (Service service : services) { - servicesList += service.getName() + " "; - } + String servicesList = ""; + for (Service service : services) { + servicesList += service.getName() + " "; + } s_logger.warn("NetScaler network element can only support LB and Static NAT services and service combination " + servicesList + " is not supported."); return false; @@ -616,8 +616,8 @@ StaticNatServiceProvider { List destinations = rule.getDestinations(); if ((destinations != null && !destinations.isEmpty()) || rule.isAutoScaleConfig()) { - LoadBalancerTO loadBalancer = new LoadBalancerTO(rule.getId(), srcIp, srcPort, protocol, algorithm, revoked, false, destinations, rule.getStickinessPolicies()); - if(rule.isAutoScaleConfig()) { + LoadBalancerTO loadBalancer = new LoadBalancerTO(rule.getUuid(), srcIp, srcPort, protocol, algorithm, revoked, false, destinations, rule.getStickinessPolicies()); + if (rule.isAutoScaleConfig()) { loadBalancer.setAutoScaleVmGroup(rule.getAutoScaleVmGroup()); } loadBalancersToApply.add(loadBalancer); diff --git a/server/src/com/cloud/network/lb/ElasticLoadBalancerManagerImpl.java b/server/src/com/cloud/network/lb/ElasticLoadBalancerManagerImpl.java index ce38e6d4a52..037ce5830a4 100644 --- a/server/src/com/cloud/network/lb/ElasticLoadBalancerManagerImpl.java +++ b/server/src/com/cloud/network/lb/ElasticLoadBalancerManagerImpl.java @@ -133,7 +133,7 @@ import com.cloud.vm.dao.NicDao; @Local(value = { ElasticLoadBalancerManager.class }) public class ElasticLoadBalancerManagerImpl implements -ElasticLoadBalancerManager, Manager, VirtualMachineGuru { + ElasticLoadBalancerManager, Manager, VirtualMachineGuru { private static final Logger s_logger = Logger .getLogger(ElasticLoadBalancerManagerImpl.class); @@ -177,9 +177,9 @@ ElasticLoadBalancerManager, Manager, VirtualMachineGuru { PodVlanMapDao _podVlanMapDao; @Inject ElasticLbVmMapDao _elbVmMapDao; - @Inject + @Inject NetworkDao _networksDao; - @Inject + @Inject AccountDao _accountDao; @Inject PhysicalNetworkServiceProviderDao _physicalProviderDao; @@ -188,7 +188,6 @@ ElasticLoadBalancerManager, Manager, VirtualMachineGuru { @Inject NicDao _nicDao; - String _name; String _instance; static final private String _elbVmNamePrefix = "l"; @@ -203,7 +202,7 @@ ElasticLoadBalancerManager, Manager, VirtualMachineGuru { String _mgmtCidr; String _mgmtHost; - Set _gcCandidateElbVmIds = Collections.newSetFromMap(new ConcurrentHashMap()); + Set _gcCandidateElbVmIds = Collections.newSetFromMap(new ConcurrentHashMap()); int _elasticLbVmRamSize; int _elasticLbvmCpuMHz; @@ -218,12 +217,11 @@ ElasticLoadBalancerManager, Manager, VirtualMachineGuru { } } - - public DomainRouterVO deployLoadBalancerVM(Long networkId, IPAddressVO ipAddr, Long accountId) { + public DomainRouterVO deployLoadBalancerVM(Long networkId, IPAddressVO ipAddr, Long accountId) { NetworkVO network = _networkDao.findById(networkId); DataCenter dc = _dcDao.findById(network.getDataCenterId()); Long podId = getPodIdForDirectIp(ipAddr); - Pod pod = podId == null?null:_podDao.findById(podId); + Pod pod = podId == null ? null : _podDao.findById(podId); Map params = new HashMap( 1); params.put(VirtualMachineProfile.Param.ReProgramGuestNetworks, true); @@ -278,7 +276,6 @@ ElasticLoadBalancerManager, Manager, VirtualMachineGuru { private void createApplyLoadBalancingRulesCommands( List rules, DomainRouterVO elbVm, Commands cmds, long guestNetworkId) { - LoadBalancerTO[] lbs = new LoadBalancerTO[rules.size()]; int i = 0; for (LoadBalancingRule rule : rules) { @@ -290,19 +287,20 @@ ElasticLoadBalancerManager, Manager, VirtualMachineGuru { String elbIp = _networkMgr.getIp(rule.getSourceIpAddressId()).getAddress() .addr(); int srcPort = rule.getSourcePortStart(); + String uuid = rule.getUuid(); List destinations = rule.getDestinations(); - LoadBalancerTO lb = new LoadBalancerTO(rule.getId(), elbIp, srcPort, protocol, algorithm, revoked, false, destinations); - lbs[i++] = lb; + LoadBalancerTO lb = new LoadBalancerTO(uuid, elbIp, srcPort, protocol, algorithm, revoked, false, destinations); + lbs[i++] = lb; } - LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(lbs,elbVm.getPublicIpAddress(), - _nicDao.getIpAddress(guestNetworkId, elbVm.getId()),elbVm.getPrivateIpAddress(), null, null); + LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(lbs, elbVm.getPublicIpAddress(), + _nicDao.getIpAddress(guestNetworkId, elbVm.getId()), elbVm.getPrivateIpAddress(), null, null); cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, elbVm.getPrivateIpAddress()); cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, elbVm.getInstanceName()); - //FIXME: why are we setting attributes directly? Ick!! There should be accessors and - //the constructor should set defaults. + // FIXME: why are we setting attributes directly? Ick!! There should be accessors and + // the constructor should set defaults. cmd.lbStatsVisibility = _configDao.getValue(Config.NetworkLBHaproxyStatsVisbility.key()); cmd.lbStatsUri = _configDao.getValue(Config.NetworkLBHaproxyStatsUri.key()); cmd.lbStatsAuth = _configDao.getValue(Config.NetworkLBHaproxyStatsAuth.key()); @@ -320,7 +318,7 @@ ElasticLoadBalancerManager, Manager, VirtualMachineGuru { return sendCommandsToRouter(elbVm, cmds); } - protected DomainRouterVO findElbVmForLb(FirewallRule lb) {//TODO: use a table to lookup + protected DomainRouterVO findElbVmForLb(FirewallRule lb) {// TODO: use a table to lookup ElasticLbVmMapVO map = _elbVmMapDao.findOneByIp(lb.getSourceIpAddressId()); if (map == null) { return null; @@ -332,7 +330,7 @@ ElasticLoadBalancerManager, Manager, VirtualMachineGuru { @Override public boolean applyLoadBalancerRules(Network network, List rules) - throws ResourceUnavailableException { + throws ResourceUnavailableException { if (rules == null || rules.isEmpty()) { return true; } @@ -351,7 +349,7 @@ ElasticLoadBalancerManager, Manager, VirtualMachineGuru { } if (elbVm.getState() == State.Running) { - //resend all rules for the public ip + // resend all rules for the public ip List lbs = _lbDao.listByIpAddress(rules.get(0).getSourceIpAddressId()); List lbRules = new ArrayList(); for (LoadBalancerVO lb : lbs) { @@ -359,7 +357,7 @@ ElasticLoadBalancerManager, Manager, VirtualMachineGuru { List policyList = _lbMgr.getStickinessPolicies(lb.getId()); LoadBalancingRule loadBalancing = new LoadBalancingRule( lb, dstList, policyList); - lbRules.add(loadBalancing); + lbRules.add(loadBalancing); } return applyLBRules(elbVm, lbRules, network.getId()); } else if (elbVm.getState() == State.Stopped @@ -395,36 +393,33 @@ ElasticLoadBalancerManager, Manager, VirtualMachineGuru { _elasticLbVmRamSize = NumbersUtil.parseInt(configs.get(Config.ElasticLoadBalancerVmMemory.key()), DEFAULT_ELB_VM_RAMSIZE); _elasticLbvmCpuMHz = NumbersUtil.parseInt(configs.get(Config.ElasticLoadBalancerVmCpuMhz.key()), DEFAULT_ELB_VM_CPU_MHZ); _elasticLbvmNumCpu = NumbersUtil.parseInt(configs.get(Config.ElasticLoadBalancerVmNumVcpu.key()), 1); - _elasticLbVmOffering = new ServiceOfferingVO("System Offering For Elastic LB VM", _elasticLbvmNumCpu, - _elasticLbVmRamSize, _elasticLbvmCpuMHz, 0, 0, true, null, useLocalStorage, + _elasticLbVmOffering = new ServiceOfferingVO("System Offering For Elastic LB VM", _elasticLbvmNumCpu, + _elasticLbVmRamSize, _elasticLbvmCpuMHz, 0, 0, true, null, useLocalStorage, true, null, true, VirtualMachine.Type.ElasticLoadBalancerVm, true); _elasticLbVmOffering.setUniqueName(ServiceOffering.elbVmDefaultOffUniqueName); _elasticLbVmOffering = _serviceOfferingDao.persistSystemServiceOffering(_elasticLbVmOffering); - - String enabled = _configDao.getValue(Config.ElasticLoadBalancerEnabled.key()); - _enabled = (enabled == null) ? false: Boolean.parseBoolean(enabled); + _enabled = (enabled == null) ? false : Boolean.parseBoolean(enabled); s_logger.info("Elastic Load balancer enabled: " + _enabled); if (_enabled) { String traffType = _configDao.getValue(Config.ElasticLoadBalancerNetwork.key()); if ("guest".equalsIgnoreCase(traffType)) { _frontendTrafficType = TrafficType.Guest; - } else if ("public".equalsIgnoreCase(traffType)){ + } else if ("public".equalsIgnoreCase(traffType)) { _frontendTrafficType = TrafficType.Public; } else throw new ConfigurationException("ELB: Traffic type for front end of load balancer has to be guest or public; found : " + traffType); - s_logger.info("ELB: Elastic Load Balancer: will balance on " + traffType ); - int gcIntervalMinutes = NumbersUtil.parseInt(configs.get(Config.ElasticLoadBalancerVmGcInterval.key()), 5); - if (gcIntervalMinutes < 5) + s_logger.info("ELB: Elastic Load Balancer: will balance on " + traffType); + int gcIntervalMinutes = NumbersUtil.parseInt(configs.get(Config.ElasticLoadBalancerVmGcInterval.key()), 5); + if (gcIntervalMinutes < 5) gcIntervalMinutes = 5; - s_logger.info("ELB: Elastic Load Balancer: scheduling GC to run every " + gcIntervalMinutes + " minutes" ); + s_logger.info("ELB: Elastic Load Balancer: scheduling GC to run every " + gcIntervalMinutes + " minutes"); _gcThreadPool = Executors.newScheduledThreadPool(1, new NamedThreadFactory("ELBVM-GC")); _gcThreadPool.scheduleAtFixedRate(new CleanupThread(), gcIntervalMinutes, gcIntervalMinutes, TimeUnit.MINUTES); _itMgr.registerGuru(VirtualMachine.Type.ElasticLoadBalancerVm, this); } - return true; } @@ -447,17 +442,17 @@ ElasticLoadBalancerManager, Manager, VirtualMachineGuru { List unusedElbVms = _elbVmMapDao.listUnusedElbVms(); if (unusedElbVms.size() > 0) { List candidateVms = new ArrayList(); - for (DomainRouterVO candidateVm: unusedElbVms) { + for (DomainRouterVO candidateVm : unusedElbVms) { if (candidateVm.getPodIdToDeployIn() == getPodIdForDirectIp(ipAddr)) candidateVms.add(candidateVm); } - return candidateVms.size()==0?null:candidateVms.get(new Random().nextInt(candidateVms.size())); + return candidateVms.size() == 0 ? null : candidateVms.get(new Random().nextInt(candidateVms.size())); } return null; } public DomainRouterVO deployELBVm(Network guestNetwork, DeployDestination dest, Account owner, Map params) throws - ConcurrentOperationException, ResourceUnavailableException, InsufficientCapacityException { + ConcurrentOperationException, ResourceUnavailableException, InsufficientCapacityException { long dcId = dest.getDataCenter().getId(); // lock guest network @@ -477,10 +472,9 @@ ElasticLoadBalancerManager, Manager, VirtualMachineGuru { if (s_logger.isDebugEnabled()) { s_logger.debug("Starting a ELB vm for network configurations: " + guestNetwork + " in " + dest); } - assert guestNetwork.getState() == Network.State.Implemented - || guestNetwork.getState() == Network.State.Setup - || guestNetwork.getState() == Network.State.Implementing - : "Network is not yet fully implemented: "+ guestNetwork; + assert guestNetwork.getState() == Network.State.Implemented + || guestNetwork.getState() == Network.State.Setup + || guestNetwork.getState() == Network.State.Implementing : "Network is not yet fully implemented: " + guestNetwork; DataCenterDeployment plan = null; DomainRouterVO elbVm = null; @@ -503,7 +497,6 @@ ElasticLoadBalancerManager, Manager, VirtualMachineGuru { networks.add(new Pair(controlConfig, null)); networks.add(new Pair((NetworkVO) guestNetwork, guestNic)); - VMTemplateVO template = _templateDao.findSystemVMTemplate(dcId); String typeString = "ElasticLoadBalancerVm"; @@ -517,13 +510,13 @@ ElasticLoadBalancerManager, Manager, VirtualMachineGuru { throw new CloudRuntimeException("Cannot find virtual router provider " + typeString + " as service provider " + provider.getId()); } - elbVm = new DomainRouterVO(id, _elasticLbVmOffering.getId(), vrProvider.getId(), + elbVm = new DomainRouterVO(id, _elasticLbVmOffering.getId(), vrProvider.getId(), VirtualMachineName.getSystemVmName(id, _instance, _elbVmNamePrefix), template.getId(), template.getHypervisorType(), template.getGuestOSId(), owner.getDomainId(), owner.getId(), false, 0, false, RedundantState.UNKNOWN, _elasticLbVmOffering.getOfferHA(), false, VirtualMachine.Type.ElasticLoadBalancerVm, null); elbVm.setRole(Role.LB); elbVm = _itMgr.allocate(elbVm, template, _elasticLbVmOffering, networks, plan, null, owner); - //TODO: create usage stats + // TODO: create usage stats } State state = elbVm.getState(); @@ -531,7 +524,6 @@ ElasticLoadBalancerManager, Manager, VirtualMachineGuru { elbVm = this.start(elbVm, _accountService.getSystemUser(), _accountService.getSystemAccount(), params); } - return elbVm; } finally { _networkDao.releaseFromLockTable(guestNetworkId); @@ -539,7 +531,7 @@ ElasticLoadBalancerManager, Manager, VirtualMachineGuru { } private DomainRouterVO start(DomainRouterVO elbVm, User user, Account caller, Map params) throws StorageUnavailableException, InsufficientCapacityException, - ConcurrentOperationException, ResourceUnavailableException { + ConcurrentOperationException, ResourceUnavailableException { s_logger.debug("Starting ELB VM " + elbVm); if (_itMgr.start(elbVm, params, user, caller) != null) { return _routerDao.findById(elbVm.getId()); @@ -548,11 +540,10 @@ ElasticLoadBalancerManager, Manager, VirtualMachineGuru { } } - private DomainRouterVO stop(DomainRouterVO elbVm, boolean forced, User user, Account caller) throws ConcurrentOperationException, ResourceUnavailableException { s_logger.debug("Stopping ELB vm " + elbVm); try { - if (_itMgr.advanceStop( elbVm, forced, user, caller)) { + if (_itMgr.advanceStop(elbVm, forced, user, caller)) { return _routerDao.findById(elbVm.getId()); } else { return null; @@ -564,7 +555,7 @@ ElasticLoadBalancerManager, Manager, VirtualMachineGuru { protected List findExistingLoadBalancers(String lbName, Long ipId, Long accountId, Long domainId, Integer publicPort) { SearchBuilder sb = _lbDao.createSearchBuilder(); - sb.and("name", sb.entity().getName(), SearchCriteria.Op.EQ); + sb.and("name", sb.entity().getName(), SearchCriteria.Op.EQ); sb.and("accountId", sb.entity().getAccountId(), SearchCriteria.Op.EQ); sb.and("publicPort", sb.entity().getSourcePortStart(), SearchCriteria.Op.EQ); if (ipId != null) { @@ -581,16 +572,16 @@ ElasticLoadBalancerManager, Manager, VirtualMachineGuru { sc.setParameters("accountId", accountId); if (ipId != null) { sc.setParameters("sourceIpAddress", ipId); - } + } if (domainId != null) { - sc.setParameters("domainId",domainId); + sc.setParameters("domainId", domainId); } if (publicPort != null) { sc.setParameters("publicPort", publicPort); } List lbs = _lbDao.search(sc, null); - return lbs == null || lbs.size()==0 ? null: lbs; + return lbs == null || lbs.size() == 0 ? null : lbs; } @DB @@ -601,7 +592,7 @@ ElasticLoadBalancerManager, Manager, VirtualMachineGuru { PublicIp ip = _networkMgr.assignPublicIpAddress(frontEndNetwork.getDataCenterId(), null, account, VlanType.DirectAttached, frontEndNetwork.getId(), null, true); IPAddressVO ipvo = _ipAddressDao.findById(ip.getId()); - ipvo.setAssociatedWithNetworkId(frontEndNetwork.getId()); + ipvo.setAssociatedWithNetworkId(frontEndNetwork.getId()); _ipAddressDao.update(ipvo.getId(), ipvo); txn.commit(); s_logger.info("Acquired frontend IP for ELB " + ip); @@ -612,7 +603,7 @@ ElasticLoadBalancerManager, Manager, VirtualMachineGuru { public void releaseIp(long ipId, long userId, Account caller) { s_logger.info("ELB: Release public IP for loadbalancing " + ipId); IPAddressVO ipvo = _ipAddressDao.findById(ipId); - ipvo.setAssociatedWithNetworkId(null); + ipvo.setAssociatedWithNetworkId(null); _ipAddressDao.update(ipvo.getId(), ipvo); _networkMgr.disassociatePublicIpAddress(ipId, userId, caller); _ipAddressDao.unassignIpAddress(ipId); @@ -620,8 +611,8 @@ ElasticLoadBalancerManager, Manager, VirtualMachineGuru { @Override @DB - public LoadBalancer handleCreateLoadBalancerRule(CreateLoadBalancerRuleCmd lb, Account account, long networkId) throws InsufficientAddressCapacityException, NetworkRuleConflictException { - //this part of code is executed when the LB provider is Elastic Load Balancer vm + public LoadBalancer handleCreateLoadBalancerRule(CreateLoadBalancerRuleCmd lb, Account account, long networkId) throws InsufficientAddressCapacityException, NetworkRuleConflictException { + // this part of code is executed when the LB provider is Elastic Load Balancer vm if (!_networkMgr.isProviderSupportServiceInNetwork(lb.getNetworkId(), Service.Lb, Provider.ElasticLoadBalancerVm)) { return null; } @@ -638,14 +629,14 @@ ElasticLoadBalancerManager, Manager, VirtualMachineGuru { } try { List existingLbs = findExistingLoadBalancers(lb.getName(), lb.getSourceIpAddressId(), lb.getAccountId(), lb.getDomainId(), lb.getSourcePortStart()); - if (existingLbs == null ){ + if (existingLbs == null) { existingLbs = findExistingLoadBalancers(lb.getName(), lb.getSourceIpAddressId(), lb.getAccountId(), lb.getDomainId(), null); if (existingLbs == null) { if (lb.getSourceIpAddressId() != null) { existingLbs = findExistingLoadBalancers(lb.getName(), null, lb.getAccountId(), lb.getDomainId(), null); if (existingLbs != null) { throw new InvalidParameterValueException("Supplied LB name " + lb.getName() + " is not associated with IP " + lb.getSourceIpAddressId(), null); - } + } } else { s_logger.debug("Could not find any existing frontend ips for this account for this LB rule, acquiring a new frontent IP for ELB"); PublicIp ip = allocDirectIp(account, networkId); @@ -678,7 +669,6 @@ ElasticLoadBalancerManager, Manager, VirtualMachineGuru { DomainRouterVO elbVm = null; - if (existingLbs == null) { elbVm = findELBVmWithCapacity(network, ipAddr); if (elbVm == null) { @@ -721,7 +711,7 @@ ElasticLoadBalancerManager, Manager, VirtualMachineGuru { if (unusedElbVms != null && unusedElbVms.size() > 0) s_logger.info("Found " + unusedElbVms.size() + " unused ELB vms"); Set currentGcCandidates = new HashSet(); - for (DomainRouterVO elbVm: unusedElbVms) { + for (DomainRouterVO elbVm : unusedElbVms) { currentGcCandidates.add(elbVm.getId()); } _gcCandidateElbVmIds.retainAll(currentGcCandidates); @@ -782,7 +772,6 @@ ElasticLoadBalancerManager, Manager, VirtualMachineGuru { } } - @Override public DomainRouterVO findByName(String name) { if (!VirtualMachineName.isValidSystemVmName(name, _instance, _elbVmNamePrefix)) { @@ -792,19 +781,16 @@ ElasticLoadBalancerManager, Manager, VirtualMachineGuru { return _routerDao.findById(VirtualMachineName.getSystemVmId(name)); } - @Override public DomainRouterVO findById(long id) { return _routerDao.findById(id); } - @Override public DomainRouterVO persist(DomainRouterVO elbVm) { return _routerDao.persist(elbVm); } - @Override public boolean finalizeVirtualMachineProfile(VirtualMachineProfile profile, DeployDestination dest, ReservationContext context) { DomainRouterVO elbVm = profile.getVirtualMachine(); @@ -841,7 +827,7 @@ ElasticLoadBalancerManager, Manager, VirtualMachineGuru { if (nic.getTrafficType() == TrafficType.Management) { buf.append(" localgw=").append(dest.getPod().getGateway()); } else if (nic.getTrafficType() == TrafficType.Control) { - // control command is sent over management network in VMware + // control command is sent over management network in VMware if (dest.getHost().getHypervisorType() == HypervisorType.VMware) { if (s_logger.isInfoEnabled()) { s_logger.info("Check if we need to add management server explicit route to ELB vm. pod cidr: " + dest.getPod().getCidrAddress() + "/" + dest.getPod().getCidrSize() @@ -868,7 +854,7 @@ ElasticLoadBalancerManager, Manager, VirtualMachineGuru { String domain = guestNetwork.getNetworkDomain(); if (domain != null) { buf.append(" domain=" + domain); - } + } buf.append(" dns1=").append(defaultDns1); if (defaultDns2 != null) { @@ -886,7 +872,6 @@ ElasticLoadBalancerManager, Manager, VirtualMachineGuru { return true; } - @Override public boolean finalizeDeployment(Commands cmds, VirtualMachineProfile profile, DeployDestination dest, ReservationContext context) throws ResourceUnavailableException { DomainRouterVO elbVm = profile.getVirtualMachine(); @@ -908,7 +893,6 @@ ElasticLoadBalancerManager, Manager, VirtualMachineGuru { return true; } - @Override public boolean finalizeStart(VirtualMachineProfile profile, long hostId, Commands cmds, ReservationContext context) { CheckSshAnswer answer = (CheckSshAnswer) cmds.getAnswer("checkSsh"); @@ -920,7 +904,6 @@ ElasticLoadBalancerManager, Manager, VirtualMachineGuru { return true; } - @Override public boolean finalizeCommandsOnStart(Commands cmds, VirtualMachineProfile profile) { DomainRouterVO elbVm = profile.getVirtualMachine(); @@ -929,7 +912,7 @@ ElasticLoadBalancerManager, Manager, VirtualMachineGuru { NicProfile controlNic = null; Long guestNetworkId = null; - if(profile.getHypervisorType() == HypervisorType.VMware && dcVo.getNetworkType() == NetworkType.Basic) { + if (profile.getHypervisorType() == HypervisorType.VMware && dcVo.getNetworkType() == NetworkType.Basic) { // TODO this is a ugly to test hypervisor type here // for basic network mode, we will use the guest NIC for control NIC for (NicProfile nic : profile.getNics()) { @@ -973,7 +956,6 @@ ElasticLoadBalancerManager, Manager, VirtualMachineGuru { return true; } - @Override public void finalizeStop(VirtualMachineProfile profile, StopAnswer answer) { if (answer != null) { @@ -984,10 +966,9 @@ ElasticLoadBalancerManager, Manager, VirtualMachineGuru { } public void processStopOrRebootAnswer(final DomainRouterVO elbVm, Answer answer) { - //TODO: process network usage stats + // TODO: process network usage stats } - @Override public void finalizeExpunge(DomainRouterVO vm) { // no-op @@ -1007,25 +988,22 @@ ElasticLoadBalancerManager, Manager, VirtualMachineGuru { public boolean plugNic(Network network, NicTO nic, VirtualMachineTO vm, ReservationContext context, DeployDestination dest) throws ConcurrentOperationException, ResourceUnavailableException, InsufficientCapacityException { - //not supported + // not supported throw new UnsupportedOperationException("Plug nic is not supported for vm of type " + vm.getType()); } - @Override public boolean unplugNic(Network network, NicTO nic, VirtualMachineTO vm, ReservationContext context, DeployDestination dest) throws ConcurrentOperationException, ResourceUnavailableException { - //not supported + // not supported throw new UnsupportedOperationException("Unplug nic is not supported for vm of type " + vm.getType()); } - @Override public void prepareStop(VirtualMachineProfile profile) { } - @Override public boolean recreateNeeded( VirtualMachineProfile profile, long hostId, diff --git a/server/src/com/cloud/network/lb/LoadBalancingRulesManagerImpl.java b/server/src/com/cloud/network/lb/LoadBalancingRulesManagerImpl.java index 00aa9f757ef..05b745f16a7 100755 --- a/server/src/com/cloud/network/lb/LoadBalancingRulesManagerImpl.java +++ b/server/src/com/cloud/network/lb/LoadBalancingRulesManagerImpl.java @@ -106,6 +106,8 @@ import com.cloud.network.vpc.VpcManager; import com.cloud.offering.NetworkOffering; import com.cloud.projects.Project.ListProjectResourcesCriteria; import com.cloud.server.ResourceTag.TaggedResourceType; +import com.cloud.service.dao.ServiceOfferingDao; +import com.cloud.storage.dao.VMTemplateDao; import com.cloud.tags.ResourceTagVO; import com.cloud.tags.dao.ResourceTagDao; import com.cloud.template.TemplateManager; @@ -194,6 +196,10 @@ public class LoadBalancingRulesManagerImpl implements LoadBalancingRulesMa @Inject ResourceTagDao _resourceTagDao; @Inject + VMTemplateDao _templateDao; + @Inject + ServiceOfferingDao _offeringsDao; + @Inject CounterDao _counterDao; @Inject ConditionDao _conditionDao; @@ -262,20 +268,24 @@ public class LoadBalancingRulesManagerImpl implements LoadBalancingRulesMa String apiKey = user.getApiKey(); String secretKey = user.getSecretKey(); String csUrl = _configDao.getValue(Config.EndpointeUrl.key()); + String zoneId = _dcDao.findById(autoScaleVmProfile.getZoneId()).getUuid(); + String domainId = _domainDao.findById(autoScaleVmProfile.getDomainId()).getUuid(); + String serviceOfferingId = _offeringsDao.findById(autoScaleVmProfile.getServiceOfferingId()).getUuid(); + String templateId = _templateDao.findById(autoScaleVmProfile.getTemplateId()).getUuid(); - if(apiKey == null) { + if (apiKey == null) { throw new InvalidParameterValueException("apiKey for user: " + user.getUsername() + " is empty. Please generate it", null); } - if(secretKey == null) { + if (secretKey == null) { throw new InvalidParameterValueException("secretKey for user: " + user.getUsername() + " is empty. Please generate it", null); } - if(csUrl == null || csUrl.contains("localhost")) { + if (csUrl == null || csUrl.contains("localhost")) { throw new InvalidParameterValueException("Global setting endpointe.url has to be set to the Management Server's API end point", null); } - LbAutoScaleVmProfile lbAutoScaleVmProfile = new LbAutoScaleVmProfile(autoScaleVmProfile, apiKey, secretKey, csUrl); + LbAutoScaleVmProfile lbAutoScaleVmProfile = new LbAutoScaleVmProfile(autoScaleVmProfile, apiKey, secretKey, csUrl, zoneId, domainId, serviceOfferingId, templateId); return new LbAutoScaleVmGroup(vmGroup, autoScalePolicies, lbAutoScaleVmProfile); } @@ -1547,7 +1557,7 @@ public class LoadBalancingRulesManagerImpl implements LoadBalancingRulesMa protected void removeLBRule(LoadBalancerVO rule) { - //remove the rule + // remove the rule _lbDao.remove(rule.getId()); // if the rule is the last one for the ip address assigned to VPC, unassign it from the network diff --git a/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java b/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java index 6f40d165d6a..bf57bc794ed 100755 --- a/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java +++ b/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java @@ -1,3474 +1,3478 @@ -// Copyright 2012 Citrix Systems, Inc. Licensed under the -// Apache License, Version 2.0 (the "License"); you may not use this -// file except in compliance with the License. Citrix Systems, Inc. -// reserves all rights not expressly granted by the License. -// You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Automatically generated by addcopyright.py at 04/03/2012 -package com.cloud.network.router; - -import java.util.ArrayList; -import java.util.Calendar; -import java.util.Collections; -import java.util.Comparator; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TimeZone; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; - -import javax.ejb.Local; -import javax.naming.ConfigurationException; - -import org.apache.log4j.Logger; - -import com.cloud.agent.AgentManager; -import com.cloud.agent.AgentManager.OnError; -import com.cloud.agent.Listener; -import com.cloud.agent.api.AgentControlAnswer; -import com.cloud.agent.api.AgentControlCommand; -import com.cloud.agent.api.Answer; -import com.cloud.agent.api.BumpUpPriorityCommand; -import com.cloud.agent.api.CheckRouterAnswer; -import com.cloud.agent.api.CheckRouterCommand; -import com.cloud.agent.api.CheckS2SVpnConnectionsAnswer; -import com.cloud.agent.api.CheckS2SVpnConnectionsCommand; -import com.cloud.agent.api.Command; -import com.cloud.agent.api.GetDomRVersionAnswer; -import com.cloud.agent.api.GetDomRVersionCmd; -import com.cloud.agent.api.ModifySshKeysCommand; -import com.cloud.agent.api.NetworkUsageAnswer; -import com.cloud.agent.api.NetworkUsageCommand; -import com.cloud.agent.api.StartupCommand; -import com.cloud.agent.api.StopAnswer; -import com.cloud.agent.api.check.CheckSshAnswer; -import com.cloud.agent.api.check.CheckSshCommand; -import com.cloud.agent.api.routing.DhcpEntryCommand; -import com.cloud.agent.api.routing.IpAssocCommand; -import com.cloud.agent.api.routing.LoadBalancerConfigCommand; -import com.cloud.agent.api.routing.NetworkElementCommand; -import com.cloud.agent.api.routing.RemoteAccessVpnCfgCommand; -import com.cloud.agent.api.routing.SavePasswordCommand; -import com.cloud.agent.api.routing.SetFirewallRulesCommand; -import com.cloud.agent.api.routing.SetPortForwardingRulesCommand; -import com.cloud.agent.api.routing.SetPortForwardingRulesVpcCommand; -import com.cloud.agent.api.routing.SetStaticNatRulesCommand; -import com.cloud.agent.api.routing.VmDataCommand; -import com.cloud.agent.api.routing.VpnUsersCfgCommand; -import com.cloud.agent.api.to.FirewallRuleTO; -import com.cloud.agent.api.to.IpAddressTO; -import com.cloud.agent.api.to.LoadBalancerTO; -import com.cloud.agent.api.to.NicTO; -import com.cloud.agent.api.to.PortForwardingRuleTO; -import com.cloud.agent.api.to.StaticNatRuleTO; -import com.cloud.agent.api.to.VirtualMachineTO; -import com.cloud.agent.manager.Commands; -import com.cloud.alert.AlertManager; -import com.cloud.api.commands.UpgradeRouterCmd; -import com.cloud.cluster.ManagementServerHostVO; -import com.cloud.cluster.ManagementServerNode; -import com.cloud.cluster.dao.ManagementServerHostDao; -import com.cloud.configuration.Config; -import com.cloud.configuration.ConfigurationManager; -import com.cloud.configuration.ZoneConfig; -import com.cloud.configuration.dao.ConfigurationDao; -import com.cloud.dc.ClusterVO; -import com.cloud.dc.DataCenter; -import com.cloud.dc.DataCenter.NetworkType; -import com.cloud.dc.DataCenterVO; -import com.cloud.dc.HostPodVO; -import com.cloud.dc.Pod; -import com.cloud.dc.dao.ClusterDao; -import com.cloud.dc.dao.DataCenterDao; -import com.cloud.dc.dao.HostPodDao; -import com.cloud.dc.dao.VlanDao; -import com.cloud.deploy.DataCenterDeployment; -import com.cloud.deploy.DeployDestination; -import com.cloud.deploy.DeploymentPlan; -import com.cloud.deploy.DeploymentPlanner.ExcludeList; -import com.cloud.event.ActionEvent; -import com.cloud.event.EventTypes; -import com.cloud.exception.AgentUnavailableException; -import com.cloud.exception.ConcurrentOperationException; -import com.cloud.exception.ConnectionException; -import com.cloud.exception.InsufficientAddressCapacityException; -import com.cloud.exception.InsufficientCapacityException; -import com.cloud.exception.InsufficientServerCapacityException; -import com.cloud.exception.InsufficientVirtualNetworkCapcityException; -import com.cloud.exception.InvalidParameterValueException; -import com.cloud.exception.OperationTimedoutException; -import com.cloud.exception.ResourceUnavailableException; -import com.cloud.exception.StorageUnavailableException; -import com.cloud.host.HostVO; -import com.cloud.host.Status; -import com.cloud.host.dao.HostDao; -import com.cloud.hypervisor.Hypervisor.HypervisorType; -import com.cloud.network.IPAddressVO; -import com.cloud.network.IpAddress; -import com.cloud.network.LoadBalancerVO; -import com.cloud.network.Network; -import com.cloud.network.Network.GuestType; -import com.cloud.network.Network.Provider; -import com.cloud.network.Network.Service; -import com.cloud.network.NetworkManager; -import com.cloud.network.NetworkVO; -import com.cloud.network.Networks.BroadcastDomainType; -import com.cloud.network.Networks.IsolationType; -import com.cloud.network.Networks.TrafficType; -import com.cloud.network.PhysicalNetworkServiceProvider; -import com.cloud.network.PublicIpAddress; -import com.cloud.network.RemoteAccessVpn; -import com.cloud.network.Site2SiteCustomerGateway; -import com.cloud.network.Site2SiteVpnConnection; -import com.cloud.network.Site2SiteVpnConnectionVO; -import com.cloud.network.Site2SiteVpnGatewayVO; -import com.cloud.network.SshKeysDistriMonitor; -import com.cloud.network.VirtualNetworkApplianceService; -import com.cloud.network.VirtualRouterProvider; -import com.cloud.network.VirtualRouterProvider.VirtualRouterProviderType; -import com.cloud.network.VpnUser; -import com.cloud.network.VpnUserVO; -import com.cloud.network.addr.PublicIp; -import com.cloud.network.dao.FirewallRulesDao; -import com.cloud.network.dao.IPAddressDao; -import com.cloud.network.dao.LoadBalancerDao; -import com.cloud.network.dao.LoadBalancerVMMapDao; -import com.cloud.network.dao.NetworkDao; -import com.cloud.network.dao.PhysicalNetworkServiceProviderDao; -import com.cloud.network.dao.RemoteAccessVpnDao; -import com.cloud.network.dao.Site2SiteCustomerGatewayDao; -import com.cloud.network.dao.Site2SiteVpnConnectionDao; -import com.cloud.network.dao.Site2SiteVpnGatewayDao; -import com.cloud.network.dao.VirtualRouterProviderDao; -import com.cloud.network.dao.VpnUserDao; -import com.cloud.network.lb.LoadBalancingRule; -import com.cloud.network.lb.LoadBalancingRule.LbDestination; -import com.cloud.network.lb.LoadBalancingRule.LbStickinessPolicy; -import com.cloud.network.lb.LoadBalancingRulesManager; -import com.cloud.network.router.VirtualRouter.RedundantState; -import com.cloud.network.router.VirtualRouter.Role; -import com.cloud.network.rules.FirewallRule; -import com.cloud.network.rules.FirewallRule.Purpose; -import com.cloud.network.rules.PortForwardingRule; -import com.cloud.network.rules.RulesManager; -import com.cloud.network.rules.StaticNat; -import com.cloud.network.rules.StaticNatImpl; -import com.cloud.network.rules.StaticNatRule; -import com.cloud.network.rules.dao.PortForwardingRulesDao; -import com.cloud.network.vpn.Site2SiteVpnManager; -import com.cloud.offering.ServiceOffering; -import com.cloud.offerings.NetworkOfferingVO; -import com.cloud.offerings.dao.NetworkOfferingDao; -import com.cloud.resource.ResourceManager; -import com.cloud.service.ServiceOfferingVO; -import com.cloud.service.dao.ServiceOfferingDao; -import com.cloud.storage.GuestOSVO; -import com.cloud.storage.VMTemplateVO; -import com.cloud.storage.Volume.Type; -import com.cloud.storage.VolumeVO; -import com.cloud.storage.dao.GuestOSDao; -import com.cloud.storage.dao.VMTemplateDao; -import com.cloud.storage.dao.VolumeDao; -import com.cloud.user.Account; -import com.cloud.user.AccountManager; -import com.cloud.user.User; -import com.cloud.user.UserContext; -import com.cloud.user.UserStatisticsVO; -import com.cloud.user.UserStatsLogVO; -import com.cloud.user.UserVO; -import com.cloud.user.dao.UserDao; -import com.cloud.user.dao.UserStatisticsDao; -import com.cloud.user.dao.UserStatsLogDao; -import com.cloud.uservm.UserVm; -import com.cloud.utils.NumbersUtil; -import com.cloud.utils.Pair; -import com.cloud.utils.PasswordGenerator; -import com.cloud.utils.StringUtils; -import com.cloud.utils.component.ComponentLocator; -import com.cloud.utils.component.Inject; -import com.cloud.utils.concurrency.NamedThreadFactory; -import com.cloud.utils.db.DB; -import com.cloud.utils.db.Filter; -import com.cloud.utils.db.GlobalLock; -import com.cloud.utils.db.Transaction; -import com.cloud.utils.exception.CloudRuntimeException; -import com.cloud.utils.net.MacAddress; -import com.cloud.utils.net.NetUtils; -import com.cloud.vm.DomainRouterVO; -import com.cloud.vm.Nic; -import com.cloud.vm.NicProfile; -import com.cloud.vm.NicVO; -import com.cloud.vm.ReservationContext; -import com.cloud.vm.ReservationContextImpl; -import com.cloud.vm.UserVmVO; -import com.cloud.vm.VMInstanceVO; -import com.cloud.vm.VirtualMachine; -import com.cloud.vm.VirtualMachine.State; -import com.cloud.vm.VirtualMachineGuru; -import com.cloud.vm.VirtualMachineManager; -import com.cloud.vm.VirtualMachineName; -import com.cloud.vm.VirtualMachineProfile; -import com.cloud.vm.VirtualMachineProfile.Param; -import com.cloud.vm.dao.DomainRouterDao; -import com.cloud.vm.dao.NicDao; -import com.cloud.vm.dao.UserVmDao; -import com.cloud.vm.dao.UserVmDetailsDao; - -/** - * VirtualNetworkApplianceManagerImpl manages the different types of virtual network appliances available in the Cloud Stack. - */ -@Local(value = { VirtualNetworkApplianceManager.class, VirtualNetworkApplianceService.class }) -public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplianceManager, VirtualNetworkApplianceService, -VirtualMachineGuru, Listener { - private static final Logger s_logger = Logger.getLogger(VirtualNetworkApplianceManagerImpl.class); - - String _name; - @Inject - DataCenterDao _dcDao = null; - @Inject - VlanDao _vlanDao = null; - @Inject - FirewallRulesDao _rulesDao = null; - @Inject - LoadBalancerDao _loadBalancerDao = null; - @Inject - LoadBalancerVMMapDao _loadBalancerVMMapDao = null; - @Inject - IPAddressDao _ipAddressDao = null; - @Inject - VMTemplateDao _templateDao = null; - @Inject - DomainRouterDao _routerDao = null; - @Inject - UserDao _userDao = null; - @Inject - UserStatisticsDao _userStatsDao = null; - @Inject - HostDao _hostDao = null; - @Inject - ConfigurationDao _configDao; - @Inject - HostPodDao _podDao = null; - @Inject - UserStatsLogDao _userStatsLogDao = null; - @Inject - AgentManager _agentMgr; - @Inject - AlertManager _alertMgr; - @Inject - AccountManager _accountMgr; - @Inject - ConfigurationManager _configMgr; - @Inject - ServiceOfferingDao _serviceOfferingDao = null; - @Inject - UserVmDao _userVmDao; - @Inject - UserStatisticsDao _statsDao = null; - @Inject - NetworkOfferingDao _networkOfferingDao = null; - @Inject - GuestOSDao _guestOSDao = null; - @Inject - NetworkManager _networkMgr; - @Inject - VirtualMachineManager _itMgr; - @Inject - VpnUserDao _vpnUsersDao; - @Inject - RemoteAccessVpnDao _remoteAccessVpnDao; - @Inject - RulesManager _rulesMgr; - @Inject - NetworkDao _networkDao; - @Inject - LoadBalancingRulesManager _lbMgr; - @Inject - PortForwardingRulesDao _pfRulesDao; - @Inject - RemoteAccessVpnDao _vpnDao; - @Inject - NicDao _nicDao; - @Inject - VolumeDao _volumeDao = null; - @Inject - UserVmDetailsDao _vmDetailsDao; - @Inject - ClusterDao _clusterDao; - @Inject - ResourceManager _resourceMgr; - @Inject - PhysicalNetworkServiceProviderDao _physicalProviderDao; - @Inject - VirtualRouterProviderDao _vrProviderDao; - @Inject - ManagementServerHostDao _msHostDao; - @Inject - Site2SiteCustomerGatewayDao _s2sCustomerGatewayDao; - @Inject - Site2SiteVpnGatewayDao _s2sVpnGatewayDao; - @Inject - Site2SiteVpnConnectionDao _s2sVpnConnectionDao; - @Inject - Site2SiteVpnManager _s2sVpnMgr; - - int _routerRamSize; - int _routerCpuMHz; - int _retry = 2; - String _instance; - String _mgmt_host; - String _mgmt_cidr; - - int _routerStatsInterval = 300; - int _routerCheckInterval = 30; - protected ServiceOfferingVO _offering; - private String _dnsBasicZoneUpdates = "all"; - - private boolean _disable_rp_filter = false; - int _routerExtraPublicNics = 2; - private int _usageAggregationRange = 1440; - private String _usageTimeZone = "GMT"; - private final long mgmtSrvrId = MacAddress.getMacAddress().toLong(); - private static final int ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_COOPERATION = 5; // 5 seconds - - ScheduledExecutorService _executor; - ScheduledExecutorService _checkExecutor; - ScheduledExecutorService _networkStatsUpdateExecutor; - - Account _systemAcct; - - @Override - public boolean sendSshKeysToHost(Long hostId, String pubKey, String prvKey) { - ModifySshKeysCommand cmd = new ModifySshKeysCommand(pubKey, prvKey); - final Answer answer = _agentMgr.easySend(hostId, cmd); - - if (answer != null) { - return true; - } else { - return false; - } - } - - @Override - public VirtualRouter destroyRouter(final long routerId) throws ResourceUnavailableException, ConcurrentOperationException { - UserContext context = UserContext.current(); - User user = _accountMgr.getActiveUser(context.getCallerUserId()); - - if (s_logger.isDebugEnabled()) { - s_logger.debug("Attempting to destroy router " + routerId); - } - - DomainRouterVO router = _routerDao.findById(routerId); - if (router == null) { - return null; - } - - _accountMgr.checkAccess(context.getCaller(), null, true, router); - - boolean result = _itMgr.expunge(router, user, _accountMgr.getAccount(router.getAccountId())); - - if (result) { - return router; - } - return null; - } - - @Override - @DB - public VirtualRouter upgradeRouter(UpgradeRouterCmd cmd) { - Long routerId = cmd.getId(); - Long serviceOfferingId = cmd.getServiceOfferingId(); - Account caller = UserContext.current().getCaller(); - - DomainRouterVO router = _routerDao.findById(routerId); - if (router == null) { - throw new InvalidParameterValueException("Unable to find router by id", null); - } - - _accountMgr.checkAccess(caller, null, true, router); - - if (router.getServiceOfferingId() == serviceOfferingId) { - s_logger.debug("Router: " + routerId + "already has service offering: " + serviceOfferingId); - return _routerDao.findById(routerId); - } - - ServiceOffering newServiceOffering = _configMgr.getServiceOffering(serviceOfferingId); - if (newServiceOffering == null) { - throw new InvalidParameterValueException("Unable to find service offering by id", null); - } - - // check if it is a system service offering, if yes return with error as it cannot be used for user vms - if (!newServiceOffering.getSystemUse()) { - throw new InvalidParameterValueException("Cannot upgrade router vm to a non system service offering " + serviceOfferingId, null); - } - - // Check that the router is stopped - if (!router.getState().equals(State.Stopped)) { - s_logger.warn("Unable to upgrade router " + router.toString() + " in state " + router.getState()); - throw new InvalidParameterValueException("Unable to upgrade router " + router.toString() + " in state " + router.getState() - + "; make sure the router is stopped and not in an error state before upgrading.", null); - } - - ServiceOfferingVO currentServiceOffering = _serviceOfferingDao.findById(router.getServiceOfferingId()); - - // Check that the service offering being upgraded to has the same storage pool preference as the VM's current service - // offering - if (currentServiceOffering.getUseLocalStorage() != newServiceOffering.getUseLocalStorage()) { - throw new InvalidParameterValueException("Can't upgrade, due to new local storage status : " + - newServiceOffering.getUseLocalStorage() + " is different from " - + "curruent local storage status: " + currentServiceOffering.getUseLocalStorage(), null); - } - - router.setServiceOfferingId(serviceOfferingId); - if (_routerDao.update(routerId, router)) { - return _routerDao.findById(routerId); - } else { - throw new CloudRuntimeException("Unable to upgrade router " + routerId); - } - - } - - @Override - public boolean savePasswordToRouter(Network network, NicProfile nic, VirtualMachineProfile profile, List routers) throws ResourceUnavailableException { - if (routers == null || routers.isEmpty()) { - s_logger.warn("Unable save password, router doesn't exist in network " + network.getId()); - throw new CloudRuntimeException("Unable to save password to router"); - } - - UserVm userVm = profile.getVirtualMachine(); - String password = (String) profile.getParameter(Param.VmPassword); - String encodedPassword = PasswordGenerator.rot13(password); - DataCenter dc = _dcDao.findById(userVm.getDataCenterIdToDeployIn()); - - boolean result = true; - for (VirtualRouter router : routers) { - boolean sendPassword = true; - if (dc.getNetworkType() == NetworkType.Basic && userVm.getPodIdToDeployIn().longValue() != router.getPodIdToDeployIn().longValue()) { - sendPassword = false; - } - - if (sendPassword) { - Commands cmds = new Commands(OnError.Continue); - SavePasswordCommand cmd = new SavePasswordCommand(encodedPassword, nic.getIp4Address(), userVm.getHostName()); - cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); - cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); - DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); - cmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); - cmds.addCommand("password", cmd); - - result = result && sendCommandsToRouter(router, cmds); - } - } - return result; - } - - @Override @ActionEvent(eventType = EventTypes.EVENT_ROUTER_STOP, eventDescription = "stopping router Vm", async = true) - public VirtualRouter stopRouter(long routerId, boolean forced) throws ResourceUnavailableException, ConcurrentOperationException { - UserContext context = UserContext.current(); - Account account = context.getCaller(); - - // verify parameters - DomainRouterVO router = _routerDao.findById(routerId); - if (router == null) { - throw new InvalidParameterValueException("Unable to find router by id", null); - } - - _accountMgr.checkAccess(account, null, true, router); - - UserVO user = _userDao.findById(UserContext.current().getCallerUserId()); - - VirtualRouter virtualRouter = stop(router, forced, user, account); - if(virtualRouter == null){ - throw new CloudRuntimeException("Failed to stop router with id " + routerId); - } - return virtualRouter; - } - - @DB - public void processStopOrRebootAnswer(final DomainRouterVO router, Answer answer) { - final Transaction txn = Transaction.currentTxn(); - try { - txn.start(); - //FIXME!!! - UserStats command should grab bytesSent/Received for all guest interfaces of the VR - List routerGuestNtwkIds = _routerDao.getRouterNetworks(router.getId()); - for (Long guestNtwkId : routerGuestNtwkIds) { - final UserStatisticsVO userStats = _userStatsDao.lock(router.getAccountId(), router.getDataCenterIdToDeployIn(), - guestNtwkId, null, router.getId(), router.getType().toString()); - if (userStats != null) { - final long currentBytesRcvd = userStats.getCurrentBytesReceived(); - userStats.setCurrentBytesReceived(0); - userStats.setNetBytesReceived(userStats.getNetBytesReceived() + currentBytesRcvd); - - final long currentBytesSent = userStats.getCurrentBytesSent(); - userStats.setCurrentBytesSent(0); - userStats.setNetBytesSent(userStats.getNetBytesSent() + currentBytesSent); - _userStatsDao.update(userStats.getId(), userStats); - s_logger.debug("Successfully updated user statistics as a part of domR " + router + " reboot/stop"); - } else { - s_logger.warn("User stats were not created for account " + router.getAccountId() + " and dc " + router.getDataCenterIdToDeployIn()); - } - } - - txn.commit(); - } catch (final Exception e) { - txn.rollback(); - throw new CloudRuntimeException("Problem updating stats after reboot/stop ", e); - } - } - - @Override @ActionEvent(eventType = EventTypes.EVENT_ROUTER_REBOOT, eventDescription = "rebooting router Vm", async = true) - public VirtualRouter rebootRouter(long routerId, boolean reprogramNetwork) throws ConcurrentOperationException, - ResourceUnavailableException, InsufficientCapacityException { - Account caller = UserContext.current().getCaller(); - - // verify parameters - DomainRouterVO router = _routerDao.findById(routerId); - if (router == null) { - throw new InvalidParameterValueException("Unable to find domain router by id", null); - } - - _accountMgr.checkAccess(caller, null, true, router); - - // Can reboot domain router only in Running state - if (router == null || router.getState() != State.Running) { - s_logger.warn("Unable to reboot, virtual router is not in the right state " + router.getState()); - throw new ResourceUnavailableException("Unable to reboot domR, it is not in right state " + router.getState(), - DataCenter.class, router.getDataCenterIdToDeployIn()); - } - - UserVO user = _userDao.findById(UserContext.current().getCallerUserId()); - s_logger.debug("Stopping and starting router " + router + " as a part of router reboot"); - - if (stop(router, false, user, caller) != null) { - return startRouter(routerId, reprogramNetwork); - } else { - throw new CloudRuntimeException("Failed to reboot router " + router); - } - } - - @Override - public boolean configure(final String name, final Map params) throws ConfigurationException { - _name = name; - - _executor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("RouterMonitor")); - _checkExecutor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("RouterStatusMonitor")); - _networkStatsUpdateExecutor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("NetworkStatsUpdater")); - - final ComponentLocator locator = ComponentLocator.getCurrentLocator(); - - final Map configs = _configDao.getConfiguration("AgentManager", params); - - _mgmt_host = configs.get("host"); - _routerRamSize = NumbersUtil.parseInt(configs.get("router.ram.size"), DEFAULT_ROUTER_VM_RAMSIZE); - _routerCpuMHz = NumbersUtil.parseInt(configs.get("router.cpu.mhz"), DEFAULT_ROUTER_CPU_MHZ); - - _routerExtraPublicNics = NumbersUtil.parseInt(_configDao.getValue(Config.RouterExtraPublicNics.key()), 2); - - String value = configs.get("start.retry"); - _retry = NumbersUtil.parseInt(value, 2); - - value = configs.get("router.stats.interval"); - _routerStatsInterval = NumbersUtil.parseInt(value, 300); - - value = configs.get("router.check.interval"); - _routerCheckInterval = NumbersUtil.parseInt(value, 30); - - _instance = configs.get("instance.name"); - if (_instance == null) { - _instance = "DEFAULT"; - } - - String rpValue = configs.get("network.disable.rpfilter"); - if (rpValue != null && rpValue.equalsIgnoreCase("true")) { - _disable_rp_filter = true; - } - - _dnsBasicZoneUpdates = String.valueOf(_configDao.getValue(Config.DnsBasicZoneUpdates.key())); - - s_logger.info("Router configurations: " + "ramsize=" + _routerRamSize); - - final UserStatisticsDao statsDao = locator.getDao(UserStatisticsDao.class); - if (statsDao == null) { - throw new ConfigurationException("Unable to get " + UserStatisticsDao.class.getName()); - } - - _agentMgr.registerForHostEvents(new SshKeysDistriMonitor(_agentMgr, _hostDao, _configDao), true, false, false); - _itMgr.registerGuru(VirtualMachine.Type.DomainRouter, this); - - boolean useLocalStorage = Boolean.parseBoolean(configs.get(Config.SystemVMUseLocalStorage.key())); - _offering = new ServiceOfferingVO("System Offering For Software Router", 1, _routerRamSize, _routerCpuMHz, null, - null, true, null, useLocalStorage, true, null, true, VirtualMachine.Type.DomainRouter, true); - _offering.setUniqueName(ServiceOffering.routerDefaultOffUniqueName); - _offering = _serviceOfferingDao.persistSystemServiceOffering(_offering); - - // this can sometimes happen, if DB is manually or programmatically manipulated - if(_offering == null) { - String msg = "Data integrity problem : System Offering For Software router VM has been removed?"; - s_logger.error(msg); - throw new ConfigurationException(msg); - } - - _systemAcct = _accountMgr.getSystemAccount(); - - String aggregationRange = configs.get("usage.stats.job.aggregation.range"); - _usageAggregationRange = NumbersUtil.parseInt(aggregationRange, 1440); - _usageTimeZone = configs.get("usage.aggregation.timezone"); - if(_usageTimeZone == null){ - _usageTimeZone = "GMT"; - } - - _agentMgr.registerForHostEvents(this, true, false, false); - - s_logger.info("DomainRouterManager is configured."); - - return true; - } - - @Override - public String getName() { - return _name; - } - - @Override - public boolean start() { - if (_routerStatsInterval > 0){ - _executor.scheduleAtFixedRate(new NetworkUsageTask(), _routerStatsInterval, _routerStatsInterval, TimeUnit.SECONDS); - }else{ - s_logger.debug("router.stats.interval - " + _routerStatsInterval+ " so not scheduling the router stats thread"); - } - - //Schedule Network stats update task - TimeZone usageTimezone = TimeZone.getTimeZone(_usageTimeZone); - Calendar cal = Calendar.getInstance(usageTimezone); - cal.setTime(new Date()); - long endDate = 0; - int HOURLY_TIME = 60; - final int DAILY_TIME = 60 * 24; - if (_usageAggregationRange == DAILY_TIME) { - cal.roll(Calendar.DAY_OF_YEAR, false); - cal.set(Calendar.HOUR_OF_DAY, 0); - cal.set(Calendar.MINUTE, 0); - cal.set(Calendar.SECOND, 0); - cal.set(Calendar.MILLISECOND, 0); - cal.roll(Calendar.DAY_OF_YEAR, true); - cal.add(Calendar.MILLISECOND, -1); - endDate = cal.getTime().getTime(); - } else if (_usageAggregationRange == HOURLY_TIME) { - cal.roll(Calendar.HOUR_OF_DAY, false); - cal.set(Calendar.MINUTE, 0); - cal.set(Calendar.SECOND, 0); - cal.set(Calendar.MILLISECOND, 0); - cal.roll(Calendar.HOUR_OF_DAY, true); - cal.add(Calendar.MILLISECOND, -1); - endDate = cal.getTime().getTime(); - } else { - endDate = cal.getTime().getTime(); - } - - _networkStatsUpdateExecutor.scheduleAtFixedRate(new NetworkStatsUpdateTask(), (endDate - System.currentTimeMillis()), - (_usageAggregationRange * 60 * 1000), TimeUnit.MILLISECONDS); - - if (_routerCheckInterval > 0) { - _checkExecutor.scheduleAtFixedRate(new CheckRouterTask(), _routerCheckInterval, _routerCheckInterval, TimeUnit.SECONDS); - } else { - s_logger.debug("router.check.interval - " + _routerCheckInterval+ " so not scheduling the redundant router checking thread"); - } - - return true; - } - - @Override - public boolean stop() { - return true; - } - - protected VirtualNetworkApplianceManagerImpl() { - } - - @Override - public Long convertToId(final String vmName) { - if (!VirtualMachineName.isValidRouterName(vmName, _instance)) { - return null; - } - - return VirtualMachineName.getRouterId(vmName); - } - - private VmDataCommand generateVmDataCommand(VirtualRouter router, String vmPrivateIpAddress, String userData, - String serviceOffering, String zoneName, String guestIpAddress, String vmName, - String vmInstanceName, long vmId, String publicKey, long guestNetworkId) { - VmDataCommand cmd = new VmDataCommand(vmPrivateIpAddress, vmName); - - cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); - cmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(guestNetworkId, router.getId())); - cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); - - DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); - cmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); - - cmd.addVmData("userdata", "user-data", userData); - cmd.addVmData("metadata", "service-offering", StringUtils.unicodeEscape(serviceOffering)); - cmd.addVmData("metadata", "availability-zone", StringUtils.unicodeEscape(zoneName)); - cmd.addVmData("metadata", "local-ipv4", guestIpAddress); - cmd.addVmData("metadata", "local-hostname", StringUtils.unicodeEscape(vmName)); - if (dcVo.getNetworkType() == NetworkType.Basic) { - cmd.addVmData("metadata", "public-ipv4", guestIpAddress); - cmd.addVmData("metadata", "public-hostname", StringUtils.unicodeEscape(vmName)); - } else - { - if (router.getPublicIpAddress() == null) { - cmd.addVmData("metadata", "public-ipv4", guestIpAddress); - } else { - cmd.addVmData("metadata", "public-ipv4", router.getPublicIpAddress()); - } - cmd.addVmData("metadata", "public-hostname", router.getPublicIpAddress()); - } - cmd.addVmData("metadata", "instance-id", vmInstanceName); - cmd.addVmData("metadata", "vm-id", String.valueOf(vmId)); - cmd.addVmData("metadata", "public-keys", publicKey); - - String cloudIdentifier = _configDao.getValue("cloud.identifier"); - if (cloudIdentifier == null) { - cloudIdentifier = ""; - } else { - cloudIdentifier = "CloudStack-{" + cloudIdentifier + "}"; - } - cmd.addVmData("metadata", "cloud-identifier", cloudIdentifier); - - return cmd; - } - - protected class NetworkUsageTask implements Runnable { - - public NetworkUsageTask() { - } - - @Override - public void run() { - try{ - final List routers = _routerDao.listByStateAndNetworkType(State.Running, GuestType.Isolated, mgmtSrvrId); - s_logger.debug("Found " + routers.size() + " running routers. "); - - for (DomainRouterVO router : routers) { - String privateIP = router.getPrivateIpAddress(); - - if (privateIP != null) { - boolean forVpc = router.getVpcId() != null; - List routerNics = _nicDao.listByVmId(router.getId()); - for (Nic routerNic : routerNics) { - Network network = _networkMgr.getNetwork(routerNic.getNetworkId()); - if ((forVpc && network.getTrafficType() == TrafficType.Public) || (!forVpc && network.getTrafficType() == TrafficType.Guest)) { - final NetworkUsageCommand usageCmd = new NetworkUsageCommand(privateIP, router.getHostName(), - forVpc, routerNic.getIp4Address()); - UserStatisticsVO previousStats = _statsDao.findBy(router.getAccountId(), - router.getDataCenterIdToDeployIn(), network.getId(), (forVpc ? routerNic.getIp4Address() : null), router.getId(), router.getType().toString()); - NetworkUsageAnswer answer = null; - try { - answer = (NetworkUsageAnswer) _agentMgr.easySend(router.getHostId(), usageCmd); - } catch (Exception e) { - s_logger.warn("Error while collecting network stats from router: "+router.getInstanceName()+" from host: "+router.getHostId(), e); - continue; - } - - if (answer != null) { - if (!answer.getResult()) { - s_logger.warn("Error while collecting network stats from router: "+router.getInstanceName()+" from host: "+router.getHostId() + "; details: " + answer.getDetails()); - continue; - } - Transaction txn = Transaction.open(Transaction.CLOUD_DB); - try { - if ((answer.getBytesReceived() == 0) && (answer.getBytesSent() == 0)) { - s_logger.debug("Recieved and Sent bytes are both 0. Not updating user_statistics"); - continue; - } - txn.start(); - UserStatisticsVO stats = _statsDao.lock(router.getAccountId(), - router.getDataCenterIdToDeployIn(), network.getId(), (forVpc ? routerNic.getIp4Address() : null), router.getId(), router.getType().toString()); - if (stats == null) { - s_logger.warn("unable to find stats for account: " + router.getAccountId()); - continue; - } - - if(previousStats != null - && ((previousStats.getCurrentBytesReceived() != stats.getCurrentBytesReceived()) - || (previousStats.getCurrentBytesSent() != stats.getCurrentBytesSent()))){ - s_logger.debug("Router stats changed from the time NetworkUsageCommand was sent. " + - "Ignoring current answer. Router: "+answer.getRouterName()+" Rcvd: " + - answer.getBytesReceived()+ "Sent: " +answer.getBytesSent()); - continue; - } - - if (stats.getCurrentBytesReceived() > answer.getBytesReceived()) { - if (s_logger.isDebugEnabled()) { - s_logger.debug("Received # of bytes that's less than the last one. " + - "Assuming something went wrong and persisting it. Router: " + - answer.getRouterName()+" Reported: " + answer.getBytesReceived() - + " Stored: " + stats.getCurrentBytesReceived()); - } - stats.setNetBytesReceived(stats.getNetBytesReceived() + stats.getCurrentBytesReceived()); - } - stats.setCurrentBytesReceived(answer.getBytesReceived()); - if (stats.getCurrentBytesSent() > answer.getBytesSent()) { - if (s_logger.isDebugEnabled()) { - s_logger.debug("Received # of bytes that's less than the last one. " + - "Assuming something went wrong and persisting it. Router: " + - answer.getRouterName()+" Reported: " + answer.getBytesSent() - + " Stored: " + stats.getCurrentBytesSent()); - } - stats.setNetBytesSent(stats.getNetBytesSent() + stats.getCurrentBytesSent()); - } - stats.setCurrentBytesSent(answer.getBytesSent()); - _statsDao.update(stats.getId(), stats); - txn.commit(); - } catch (Exception e) { - txn.rollback(); - s_logger.warn("Unable to update user statistics for account: " + router.getAccountId() - + " Rx: " + answer.getBytesReceived() + "; Tx: " + answer.getBytesSent()); - } finally { - txn.close(); - } - } - if(forVpc){ - //Get VPN gateway - Site2SiteVpnGatewayVO s2sVpn = _s2sVpnGatewayDao.findByVpcId(router.getVpcId()); - if(s2sVpn != null){ - final NetworkUsageCommand vpnUsageCmd = new NetworkUsageCommand(privateIP, router.getHostName(), "vpn", forVpc, routerNic.getIp4Address()); - previousStats = _statsDao.findBy(s2sVpn.getAccountId(), router.getDataCenterIdToDeployIn(), network.getId(), - routerNic.getIp4Address(), s2sVpn.getId(), "VPNGateway"); - answer = null; - try { - answer = (NetworkUsageAnswer) _agentMgr.easySend(router.getHostId(), vpnUsageCmd); - } catch (Exception e) { - s_logger.warn("Error while collecting vpn network stats from router: "+router.getInstanceName()+" from host: "+router.getHostId(), e); - continue; - } - - if (answer != null) { - if (!answer.getResult()) { - s_logger.warn("Error while collecting vpn network stats from router: "+router.getInstanceName()+" from host: "+router.getHostId() + "; details: " + answer.getDetails()); - continue; - } - Transaction txn = Transaction.open(Transaction.CLOUD_DB); - try { - if ((answer.getBytesReceived() == 0) && (answer.getBytesSent() == 0)) { - s_logger.debug("Recieved and Sent bytes are both 0. Not updating user_statistics"); - continue; - } - txn.start(); - UserStatisticsVO stats = _statsDao.lock(s2sVpn.getAccountId(), router.getDataCenterIdToDeployIn(), network.getId(), - routerNic.getIp4Address(), s2sVpn.getId(), "VPNGateway"); - if (stats == null) { - s_logger.warn("unable to find vpn stats for account: " + router.getAccountId()+" vpc Id: "+router.getVpcId()); - continue; - } - - if(previousStats != null - && ((previousStats.getCurrentBytesReceived() != stats.getCurrentBytesReceived()) - || (previousStats.getCurrentBytesSent() != stats.getCurrentBytesSent()))){ - s_logger.debug("Router stats changed from the time NetworkUsageCommand was sent. " + - "Ignoring current answer. Router: "+answer.getRouterName()+" Rcvd: " + - answer.getBytesReceived()+ "Sent: " +answer.getBytesSent()); - continue; - } - - if (stats.getCurrentBytesReceived() > answer.getBytesReceived()) { - if (s_logger.isDebugEnabled()) { - s_logger.debug("Received # of bytes that's less than the last one. " + - "Assuming something went wrong and persisting it. Router: " + - answer.getRouterName()+" Reported: " + answer.getBytesReceived() - + " Stored: " + stats.getCurrentBytesReceived()); - } - stats.setNetBytesReceived(stats.getNetBytesReceived() + stats.getCurrentBytesReceived()); - } - stats.setCurrentBytesReceived(answer.getBytesReceived()); - if (stats.getCurrentBytesSent() > answer.getBytesSent()) { - if (s_logger.isDebugEnabled()) { - s_logger.debug("Received # of bytes that's less than the last one. " + - "Assuming something went wrong and persisting it. Router: " + - answer.getRouterName()+" Reported: " + answer.getBytesSent() - + " Stored: " + stats.getCurrentBytesSent()); - } - stats.setNetBytesSent(stats.getNetBytesSent() + stats.getCurrentBytesSent()); - } - stats.setCurrentBytesSent(answer.getBytesSent()); - _statsDao.update(stats.getId(), stats); - txn.commit(); - } catch (Exception e) { - txn.rollback(); - s_logger.warn("Unable to update user statistics for account: " + router.getAccountId() - + " Rx: " + answer.getBytesReceived() + "; Tx: " + answer.getBytesSent()); - } finally { - txn.close(); - } - } - } - } - } - } - } - } - } catch (Exception e) { - s_logger.warn("Error while collecting network stats", e); - } - } - } - - protected class NetworkStatsUpdateTask implements Runnable { - - public NetworkStatsUpdateTask() { - } - - @Override - public void run() { - GlobalLock scanLock = GlobalLock.getInternLock("network.stats"); - try { - if(scanLock.lock(ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_COOPERATION)) { - //Check for ownership - //msHost in UP state with min id should run the job - ManagementServerHostVO msHost = _msHostDao.findOneInUpState(new Filter(ManagementServerHostVO.class, "id", true, 0L, 1L)); - if(msHost == null || (msHost.getMsid() != mgmtSrvrId)){ - s_logger.debug("Skipping aggregate network stats update"); - scanLock.unlock(); - return; - } - Transaction txn = Transaction.open(Transaction.CLOUD_DB); - try { - txn.start(); - //get all stats with delta > 0 - List updatedStats = _statsDao.listUpdatedStats(); - Date updatedTime = new Date(); - for(UserStatisticsVO stat : updatedStats){ - //update agg bytes - stat.setAggBytesReceived(stat.getCurrentBytesReceived() + stat.getNetBytesReceived()); - stat.setAggBytesSent(stat.getCurrentBytesSent() + stat.getNetBytesSent()); - _userStatsDao.update(stat.getId(), stat); - //insert into op_user_stats_log - UserStatsLogVO statsLog = new UserStatsLogVO(stat.getId(), stat.getNetBytesReceived(), stat.getNetBytesSent(), stat.getCurrentBytesReceived(), - stat.getCurrentBytesSent(), stat.getAggBytesReceived(), stat.getAggBytesSent(), updatedTime); - _userStatsLogDao.persist(statsLog); - } - s_logger.debug("Successfully updated aggregate network stats"); - txn.commit(); - } catch (Exception e){ - txn.rollback(); - s_logger.debug("Failed to update aggregate network stats", e); - } finally { - scanLock.unlock(); - txn.close(); - } - } - } catch (Exception e){ - s_logger.debug("Exception while trying to acquire network stats lock", e); - } finally { - scanLock.releaseRef(); - } - } - } - - @DB - protected void updateSite2SiteVpnConnectionState(List routers) { - for (DomainRouterVO router : routers) { - List conns = _s2sVpnMgr.getConnectionsForRouter(router); - if (conns == null || conns.isEmpty()) { - continue; - } - if (router.getState() != State.Running) { - for (Site2SiteVpnConnectionVO conn : conns) { - if (conn.getState() != Site2SiteVpnConnection.State.Error) { - conn.setState(Site2SiteVpnConnection.State.Disconnected); - _s2sVpnConnectionDao.persist(conn); - } - } - continue; - } - List ipList = new ArrayList(); - for (Site2SiteVpnConnectionVO conn : conns) { - if (conn.getState() != Site2SiteVpnConnection.State.Connected && - conn.getState() != Site2SiteVpnConnection.State.Disconnected) { - continue; - } - Site2SiteCustomerGateway gw = _s2sCustomerGatewayDao.findById(conn.getCustomerGatewayId()); - ipList.add(gw.getGatewayIp()); - } - String privateIP = router.getPrivateIpAddress(); - HostVO host = _hostDao.findById(router.getHostId()); - if (host == null || host.getStatus() != Status.Up) { - continue; - } else if (host.getManagementServerId() != ManagementServerNode.getManagementServerId()) { - /* Only cover hosts managed by this management server */ - continue; - } else if (privateIP != null) { - final CheckS2SVpnConnectionsCommand command = new CheckS2SVpnConnectionsCommand(ipList); - command.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); - command.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); - command.setWait(30); - final Answer origAnswer = _agentMgr.easySend(router.getHostId(), command); - CheckS2SVpnConnectionsAnswer answer = null; - if (origAnswer instanceof CheckS2SVpnConnectionsAnswer) { - answer = (CheckS2SVpnConnectionsAnswer)origAnswer; - } else { - s_logger.warn("Unable to update router " + router.getHostName() + "'s VPN connection status"); - continue; - } - if (!answer.getResult()) { - s_logger.warn("Unable to update router " + router.getHostName() + "'s VPN connection status"); - continue; - } - for (Site2SiteVpnConnectionVO conn : conns) { +// Copyright 2012 Citrix Systems, Inc. Licensed under the +// Apache License, Version 2.0 (the "License"); you may not use this +// file except in compliance with the License. Citrix Systems, Inc. +// reserves all rights not expressly granted by the License. +// You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Automatically generated by addcopyright.py at 04/03/2012 +package com.cloud.network.router; + +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Collections; +import java.util.Comparator; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TimeZone; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import javax.ejb.Local; +import javax.naming.ConfigurationException; + +import org.apache.log4j.Logger; + +import com.cloud.agent.AgentManager; +import com.cloud.agent.AgentManager.OnError; +import com.cloud.agent.Listener; +import com.cloud.agent.api.AgentControlAnswer; +import com.cloud.agent.api.AgentControlCommand; +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.BumpUpPriorityCommand; +import com.cloud.agent.api.CheckRouterAnswer; +import com.cloud.agent.api.CheckRouterCommand; +import com.cloud.agent.api.CheckS2SVpnConnectionsAnswer; +import com.cloud.agent.api.CheckS2SVpnConnectionsCommand; +import com.cloud.agent.api.Command; +import com.cloud.agent.api.GetDomRVersionAnswer; +import com.cloud.agent.api.GetDomRVersionCmd; +import com.cloud.agent.api.ModifySshKeysCommand; +import com.cloud.agent.api.NetworkUsageAnswer; +import com.cloud.agent.api.NetworkUsageCommand; +import com.cloud.agent.api.StartupCommand; +import com.cloud.agent.api.StopAnswer; +import com.cloud.agent.api.check.CheckSshAnswer; +import com.cloud.agent.api.check.CheckSshCommand; +import com.cloud.agent.api.routing.DhcpEntryCommand; +import com.cloud.agent.api.routing.IpAssocCommand; +import com.cloud.agent.api.routing.LoadBalancerConfigCommand; +import com.cloud.agent.api.routing.NetworkElementCommand; +import com.cloud.agent.api.routing.RemoteAccessVpnCfgCommand; +import com.cloud.agent.api.routing.SavePasswordCommand; +import com.cloud.agent.api.routing.SetFirewallRulesCommand; +import com.cloud.agent.api.routing.SetPortForwardingRulesCommand; +import com.cloud.agent.api.routing.SetPortForwardingRulesVpcCommand; +import com.cloud.agent.api.routing.SetStaticNatRulesCommand; +import com.cloud.agent.api.routing.VmDataCommand; +import com.cloud.agent.api.routing.VpnUsersCfgCommand; +import com.cloud.agent.api.to.FirewallRuleTO; +import com.cloud.agent.api.to.IpAddressTO; +import com.cloud.agent.api.to.LoadBalancerTO; +import com.cloud.agent.api.to.NicTO; +import com.cloud.agent.api.to.PortForwardingRuleTO; +import com.cloud.agent.api.to.StaticNatRuleTO; +import com.cloud.agent.api.to.VirtualMachineTO; +import com.cloud.agent.manager.Commands; +import com.cloud.alert.AlertManager; +import com.cloud.api.commands.UpgradeRouterCmd; +import com.cloud.cluster.ManagementServerHostVO; +import com.cloud.cluster.ManagementServerNode; +import com.cloud.cluster.dao.ManagementServerHostDao; +import com.cloud.configuration.Config; +import com.cloud.configuration.ConfigurationManager; +import com.cloud.configuration.ZoneConfig; +import com.cloud.configuration.dao.ConfigurationDao; +import com.cloud.dc.ClusterVO; +import com.cloud.dc.DataCenter; +import com.cloud.dc.DataCenter.NetworkType; +import com.cloud.dc.DataCenterVO; +import com.cloud.dc.HostPodVO; +import com.cloud.dc.Pod; +import com.cloud.dc.dao.ClusterDao; +import com.cloud.dc.dao.DataCenterDao; +import com.cloud.dc.dao.HostPodDao; +import com.cloud.dc.dao.VlanDao; +import com.cloud.deploy.DataCenterDeployment; +import com.cloud.deploy.DeployDestination; +import com.cloud.deploy.DeploymentPlan; +import com.cloud.deploy.DeploymentPlanner.ExcludeList; +import com.cloud.event.ActionEvent; +import com.cloud.event.EventTypes; +import com.cloud.exception.AgentUnavailableException; +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.ConnectionException; +import com.cloud.exception.InsufficientAddressCapacityException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.InsufficientServerCapacityException; +import com.cloud.exception.InsufficientVirtualNetworkCapcityException; +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.exception.OperationTimedoutException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.exception.StorageUnavailableException; +import com.cloud.host.HostVO; +import com.cloud.host.Status; +import com.cloud.host.dao.HostDao; +import com.cloud.hypervisor.Hypervisor.HypervisorType; +import com.cloud.network.IPAddressVO; +import com.cloud.network.IpAddress; +import com.cloud.network.LoadBalancerVO; +import com.cloud.network.Network; +import com.cloud.network.Network.GuestType; +import com.cloud.network.Network.Provider; +import com.cloud.network.Network.Service; +import com.cloud.network.NetworkManager; +import com.cloud.network.NetworkVO; +import com.cloud.network.Networks.BroadcastDomainType; +import com.cloud.network.Networks.IsolationType; +import com.cloud.network.Networks.TrafficType; +import com.cloud.network.PhysicalNetworkServiceProvider; +import com.cloud.network.PublicIpAddress; +import com.cloud.network.RemoteAccessVpn; +import com.cloud.network.Site2SiteCustomerGateway; +import com.cloud.network.Site2SiteVpnConnection; +import com.cloud.network.Site2SiteVpnConnectionVO; +import com.cloud.network.Site2SiteVpnGatewayVO; +import com.cloud.network.SshKeysDistriMonitor; +import com.cloud.network.VirtualNetworkApplianceService; +import com.cloud.network.VirtualRouterProvider; +import com.cloud.network.VirtualRouterProvider.VirtualRouterProviderType; +import com.cloud.network.VpnUser; +import com.cloud.network.VpnUserVO; +import com.cloud.network.addr.PublicIp; +import com.cloud.network.dao.FirewallRulesDao; +import com.cloud.network.dao.IPAddressDao; +import com.cloud.network.dao.LoadBalancerDao; +import com.cloud.network.dao.LoadBalancerVMMapDao; +import com.cloud.network.dao.NetworkDao; +import com.cloud.network.dao.PhysicalNetworkServiceProviderDao; +import com.cloud.network.dao.RemoteAccessVpnDao; +import com.cloud.network.dao.Site2SiteCustomerGatewayDao; +import com.cloud.network.dao.Site2SiteVpnConnectionDao; +import com.cloud.network.dao.Site2SiteVpnGatewayDao; +import com.cloud.network.dao.VirtualRouterProviderDao; +import com.cloud.network.dao.VpnUserDao; +import com.cloud.network.lb.LoadBalancingRule; +import com.cloud.network.lb.LoadBalancingRule.LbDestination; +import com.cloud.network.lb.LoadBalancingRule.LbStickinessPolicy; +import com.cloud.network.lb.LoadBalancingRulesManager; +import com.cloud.network.router.VirtualRouter.RedundantState; +import com.cloud.network.router.VirtualRouter.Role; +import com.cloud.network.rules.FirewallRule; +import com.cloud.network.rules.FirewallRule.Purpose; +import com.cloud.network.rules.PortForwardingRule; +import com.cloud.network.rules.RulesManager; +import com.cloud.network.rules.StaticNat; +import com.cloud.network.rules.StaticNatImpl; +import com.cloud.network.rules.StaticNatRule; +import com.cloud.network.rules.dao.PortForwardingRulesDao; +import com.cloud.network.vpn.Site2SiteVpnManager; +import com.cloud.offering.ServiceOffering; +import com.cloud.offerings.NetworkOfferingVO; +import com.cloud.offerings.dao.NetworkOfferingDao; +import com.cloud.resource.ResourceManager; +import com.cloud.service.ServiceOfferingVO; +import com.cloud.service.dao.ServiceOfferingDao; +import com.cloud.storage.GuestOSVO; +import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.Volume.Type; +import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.GuestOSDao; +import com.cloud.storage.dao.VMTemplateDao; +import com.cloud.storage.dao.VolumeDao; +import com.cloud.user.Account; +import com.cloud.user.AccountManager; +import com.cloud.user.User; +import com.cloud.user.UserContext; +import com.cloud.user.UserStatisticsVO; +import com.cloud.user.UserStatsLogVO; +import com.cloud.user.UserVO; +import com.cloud.user.dao.UserDao; +import com.cloud.user.dao.UserStatisticsDao; +import com.cloud.user.dao.UserStatsLogDao; +import com.cloud.uservm.UserVm; +import com.cloud.utils.NumbersUtil; +import com.cloud.utils.Pair; +import com.cloud.utils.PasswordGenerator; +import com.cloud.utils.StringUtils; +import com.cloud.utils.component.ComponentLocator; +import com.cloud.utils.component.Inject; +import com.cloud.utils.concurrency.NamedThreadFactory; +import com.cloud.utils.db.DB; +import com.cloud.utils.db.Filter; +import com.cloud.utils.db.GlobalLock; +import com.cloud.utils.db.Transaction; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.net.MacAddress; +import com.cloud.utils.net.NetUtils; +import com.cloud.vm.DomainRouterVO; +import com.cloud.vm.Nic; +import com.cloud.vm.NicProfile; +import com.cloud.vm.NicVO; +import com.cloud.vm.ReservationContext; +import com.cloud.vm.ReservationContextImpl; +import com.cloud.vm.UserVmVO; +import com.cloud.vm.VMInstanceVO; +import com.cloud.vm.VirtualMachine; +import com.cloud.vm.VirtualMachine.State; +import com.cloud.vm.VirtualMachineGuru; +import com.cloud.vm.VirtualMachineManager; +import com.cloud.vm.VirtualMachineName; +import com.cloud.vm.VirtualMachineProfile; +import com.cloud.vm.VirtualMachineProfile.Param; +import com.cloud.vm.dao.DomainRouterDao; +import com.cloud.vm.dao.NicDao; +import com.cloud.vm.dao.UserVmDao; +import com.cloud.vm.dao.UserVmDetailsDao; + +/** + * VirtualNetworkApplianceManagerImpl manages the different types of virtual network appliances available in the Cloud + * Stack. + */ +@Local(value = { VirtualNetworkApplianceManager.class, VirtualNetworkApplianceService.class }) +public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplianceManager, VirtualNetworkApplianceService, + VirtualMachineGuru, Listener { + private static final Logger s_logger = Logger.getLogger(VirtualNetworkApplianceManagerImpl.class); + + String _name; + @Inject + DataCenterDao _dcDao = null; + @Inject + VlanDao _vlanDao = null; + @Inject + FirewallRulesDao _rulesDao = null; + @Inject + LoadBalancerDao _loadBalancerDao = null; + @Inject + LoadBalancerVMMapDao _loadBalancerVMMapDao = null; + @Inject + IPAddressDao _ipAddressDao = null; + @Inject + VMTemplateDao _templateDao = null; + @Inject + DomainRouterDao _routerDao = null; + @Inject + UserDao _userDao = null; + @Inject + UserStatisticsDao _userStatsDao = null; + @Inject + HostDao _hostDao = null; + @Inject + ConfigurationDao _configDao; + @Inject + HostPodDao _podDao = null; + @Inject + UserStatsLogDao _userStatsLogDao = null; + @Inject + AgentManager _agentMgr; + @Inject + AlertManager _alertMgr; + @Inject + AccountManager _accountMgr; + @Inject + ConfigurationManager _configMgr; + @Inject + ServiceOfferingDao _serviceOfferingDao = null; + @Inject + UserVmDao _userVmDao; + @Inject + UserStatisticsDao _statsDao = null; + @Inject + NetworkOfferingDao _networkOfferingDao = null; + @Inject + GuestOSDao _guestOSDao = null; + @Inject + NetworkManager _networkMgr; + @Inject + VirtualMachineManager _itMgr; + @Inject + VpnUserDao _vpnUsersDao; + @Inject + RemoteAccessVpnDao _remoteAccessVpnDao; + @Inject + RulesManager _rulesMgr; + @Inject + NetworkDao _networkDao; + @Inject + LoadBalancingRulesManager _lbMgr; + @Inject + PortForwardingRulesDao _pfRulesDao; + @Inject + RemoteAccessVpnDao _vpnDao; + @Inject + NicDao _nicDao; + @Inject + VolumeDao _volumeDao = null; + @Inject + UserVmDetailsDao _vmDetailsDao; + @Inject + ClusterDao _clusterDao; + @Inject + ResourceManager _resourceMgr; + @Inject + PhysicalNetworkServiceProviderDao _physicalProviderDao; + @Inject + VirtualRouterProviderDao _vrProviderDao; + @Inject + ManagementServerHostDao _msHostDao; + @Inject + Site2SiteCustomerGatewayDao _s2sCustomerGatewayDao; + @Inject + Site2SiteVpnGatewayDao _s2sVpnGatewayDao; + @Inject + Site2SiteVpnConnectionDao _s2sVpnConnectionDao; + @Inject + Site2SiteVpnManager _s2sVpnMgr; + + int _routerRamSize; + int _routerCpuMHz; + int _retry = 2; + String _instance; + String _mgmt_host; + String _mgmt_cidr; + + int _routerStatsInterval = 300; + int _routerCheckInterval = 30; + protected ServiceOfferingVO _offering; + private String _dnsBasicZoneUpdates = "all"; + + private boolean _disable_rp_filter = false; + int _routerExtraPublicNics = 2; + private int _usageAggregationRange = 1440; + private String _usageTimeZone = "GMT"; + private final long mgmtSrvrId = MacAddress.getMacAddress().toLong(); + private static final int ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_COOPERATION = 5; // 5 seconds + + ScheduledExecutorService _executor; + ScheduledExecutorService _checkExecutor; + ScheduledExecutorService _networkStatsUpdateExecutor; + + Account _systemAcct; + + @Override + public boolean sendSshKeysToHost(Long hostId, String pubKey, String prvKey) { + ModifySshKeysCommand cmd = new ModifySshKeysCommand(pubKey, prvKey); + final Answer answer = _agentMgr.easySend(hostId, cmd); + + if (answer != null) { + return true; + } else { + return false; + } + } + + @Override + public VirtualRouter destroyRouter(final long routerId) throws ResourceUnavailableException, ConcurrentOperationException { + UserContext context = UserContext.current(); + User user = _accountMgr.getActiveUser(context.getCallerUserId()); + + if (s_logger.isDebugEnabled()) { + s_logger.debug("Attempting to destroy router " + routerId); + } + + DomainRouterVO router = _routerDao.findById(routerId); + if (router == null) { + return null; + } + + _accountMgr.checkAccess(context.getCaller(), null, true, router); + + boolean result = _itMgr.expunge(router, user, _accountMgr.getAccount(router.getAccountId())); + + if (result) { + return router; + } + return null; + } + + @Override + @DB + public VirtualRouter upgradeRouter(UpgradeRouterCmd cmd) { + Long routerId = cmd.getId(); + Long serviceOfferingId = cmd.getServiceOfferingId(); + Account caller = UserContext.current().getCaller(); + + DomainRouterVO router = _routerDao.findById(routerId); + if (router == null) { + throw new InvalidParameterValueException("Unable to find router by id", null); + } + + _accountMgr.checkAccess(caller, null, true, router); + + if (router.getServiceOfferingId() == serviceOfferingId) { + s_logger.debug("Router: " + routerId + "already has service offering: " + serviceOfferingId); + return _routerDao.findById(routerId); + } + + ServiceOffering newServiceOffering = _configMgr.getServiceOffering(serviceOfferingId); + if (newServiceOffering == null) { + throw new InvalidParameterValueException("Unable to find service offering by id", null); + } + + // check if it is a system service offering, if yes return with error as it cannot be used for user vms + if (!newServiceOffering.getSystemUse()) { + throw new InvalidParameterValueException("Cannot upgrade router vm to a non system service offering " + serviceOfferingId, null); + } + + // Check that the router is stopped + if (!router.getState().equals(State.Stopped)) { + s_logger.warn("Unable to upgrade router " + router.toString() + " in state " + router.getState()); + throw new InvalidParameterValueException("Unable to upgrade router " + router.toString() + " in state " + router.getState() + + "; make sure the router is stopped and not in an error state before upgrading.", null); + } + + ServiceOfferingVO currentServiceOffering = _serviceOfferingDao.findById(router.getServiceOfferingId()); + + // Check that the service offering being upgraded to has the same storage pool preference as the VM's current +// service + // offering + if (currentServiceOffering.getUseLocalStorage() != newServiceOffering.getUseLocalStorage()) { + throw new InvalidParameterValueException("Can't upgrade, due to new local storage status : " + + newServiceOffering.getUseLocalStorage() + " is different from " + + "curruent local storage status: " + currentServiceOffering.getUseLocalStorage(), null); + } + + router.setServiceOfferingId(serviceOfferingId); + if (_routerDao.update(routerId, router)) { + return _routerDao.findById(routerId); + } else { + throw new CloudRuntimeException("Unable to upgrade router " + routerId); + } + + } + + @Override + public boolean savePasswordToRouter(Network network, NicProfile nic, VirtualMachineProfile profile, List routers) throws ResourceUnavailableException { + if (routers == null || routers.isEmpty()) { + s_logger.warn("Unable save password, router doesn't exist in network " + network.getId()); + throw new CloudRuntimeException("Unable to save password to router"); + } + + UserVm userVm = profile.getVirtualMachine(); + String password = (String) profile.getParameter(Param.VmPassword); + String encodedPassword = PasswordGenerator.rot13(password); + DataCenter dc = _dcDao.findById(userVm.getDataCenterIdToDeployIn()); + + boolean result = true; + for (VirtualRouter router : routers) { + boolean sendPassword = true; + if (dc.getNetworkType() == NetworkType.Basic && userVm.getPodIdToDeployIn().longValue() != router.getPodIdToDeployIn().longValue()) { + sendPassword = false; + } + + if (sendPassword) { + Commands cmds = new Commands(OnError.Continue); + SavePasswordCommand cmd = new SavePasswordCommand(encodedPassword, nic.getIp4Address(), userVm.getHostName()); + cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); + cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); + DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); + cmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); + cmds.addCommand("password", cmd); + + result = result && sendCommandsToRouter(router, cmds); + } + } + return result; + } + + @Override + @ActionEvent(eventType = EventTypes.EVENT_ROUTER_STOP, eventDescription = "stopping router Vm", async = true) + public VirtualRouter stopRouter(long routerId, boolean forced) throws ResourceUnavailableException, ConcurrentOperationException { + UserContext context = UserContext.current(); + Account account = context.getCaller(); + + // verify parameters + DomainRouterVO router = _routerDao.findById(routerId); + if (router == null) { + throw new InvalidParameterValueException("Unable to find router by id", null); + } + + _accountMgr.checkAccess(account, null, true, router); + + UserVO user = _userDao.findById(UserContext.current().getCallerUserId()); + + VirtualRouter virtualRouter = stop(router, forced, user, account); + if (virtualRouter == null) { + throw new CloudRuntimeException("Failed to stop router with id " + routerId); + } + return virtualRouter; + } + + @DB + public void processStopOrRebootAnswer(final DomainRouterVO router, Answer answer) { + final Transaction txn = Transaction.currentTxn(); + try { + txn.start(); + // FIXME!!! - UserStats command should grab bytesSent/Received for all guest interfaces of the VR + List routerGuestNtwkIds = _routerDao.getRouterNetworks(router.getId()); + for (Long guestNtwkId : routerGuestNtwkIds) { + final UserStatisticsVO userStats = _userStatsDao.lock(router.getAccountId(), router.getDataCenterIdToDeployIn(), + guestNtwkId, null, router.getId(), router.getType().toString()); + if (userStats != null) { + final long currentBytesRcvd = userStats.getCurrentBytesReceived(); + userStats.setCurrentBytesReceived(0); + userStats.setNetBytesReceived(userStats.getNetBytesReceived() + currentBytesRcvd); + + final long currentBytesSent = userStats.getCurrentBytesSent(); + userStats.setCurrentBytesSent(0); + userStats.setNetBytesSent(userStats.getNetBytesSent() + currentBytesSent); + _userStatsDao.update(userStats.getId(), userStats); + s_logger.debug("Successfully updated user statistics as a part of domR " + router + " reboot/stop"); + } else { + s_logger.warn("User stats were not created for account " + router.getAccountId() + " and dc " + router.getDataCenterIdToDeployIn()); + } + } + + txn.commit(); + } catch (final Exception e) { + txn.rollback(); + throw new CloudRuntimeException("Problem updating stats after reboot/stop ", e); + } + } + + @Override + @ActionEvent(eventType = EventTypes.EVENT_ROUTER_REBOOT, eventDescription = "rebooting router Vm", async = true) + public VirtualRouter rebootRouter(long routerId, boolean reprogramNetwork) throws ConcurrentOperationException, + ResourceUnavailableException, InsufficientCapacityException { + Account caller = UserContext.current().getCaller(); + + // verify parameters + DomainRouterVO router = _routerDao.findById(routerId); + if (router == null) { + throw new InvalidParameterValueException("Unable to find domain router by id", null); + } + + _accountMgr.checkAccess(caller, null, true, router); + + // Can reboot domain router only in Running state + if (router == null || router.getState() != State.Running) { + s_logger.warn("Unable to reboot, virtual router is not in the right state " + router.getState()); + throw new ResourceUnavailableException("Unable to reboot domR, it is not in right state " + router.getState(), + DataCenter.class, router.getDataCenterIdToDeployIn()); + } + + UserVO user = _userDao.findById(UserContext.current().getCallerUserId()); + s_logger.debug("Stopping and starting router " + router + " as a part of router reboot"); + + if (stop(router, false, user, caller) != null) { + return startRouter(routerId, reprogramNetwork); + } else { + throw new CloudRuntimeException("Failed to reboot router " + router); + } + } + + @Override + public boolean configure(final String name, final Map params) throws ConfigurationException { + _name = name; + + _executor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("RouterMonitor")); + _checkExecutor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("RouterStatusMonitor")); + _networkStatsUpdateExecutor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("NetworkStatsUpdater")); + + final ComponentLocator locator = ComponentLocator.getCurrentLocator(); + + final Map configs = _configDao.getConfiguration("AgentManager", params); + + _mgmt_host = configs.get("host"); + _routerRamSize = NumbersUtil.parseInt(configs.get("router.ram.size"), DEFAULT_ROUTER_VM_RAMSIZE); + _routerCpuMHz = NumbersUtil.parseInt(configs.get("router.cpu.mhz"), DEFAULT_ROUTER_CPU_MHZ); + + _routerExtraPublicNics = NumbersUtil.parseInt(_configDao.getValue(Config.RouterExtraPublicNics.key()), 2); + + String value = configs.get("start.retry"); + _retry = NumbersUtil.parseInt(value, 2); + + value = configs.get("router.stats.interval"); + _routerStatsInterval = NumbersUtil.parseInt(value, 300); + + value = configs.get("router.check.interval"); + _routerCheckInterval = NumbersUtil.parseInt(value, 30); + + _instance = configs.get("instance.name"); + if (_instance == null) { + _instance = "DEFAULT"; + } + + String rpValue = configs.get("network.disable.rpfilter"); + if (rpValue != null && rpValue.equalsIgnoreCase("true")) { + _disable_rp_filter = true; + } + + _dnsBasicZoneUpdates = String.valueOf(_configDao.getValue(Config.DnsBasicZoneUpdates.key())); + + s_logger.info("Router configurations: " + "ramsize=" + _routerRamSize); + + final UserStatisticsDao statsDao = locator.getDao(UserStatisticsDao.class); + if (statsDao == null) { + throw new ConfigurationException("Unable to get " + UserStatisticsDao.class.getName()); + } + + _agentMgr.registerForHostEvents(new SshKeysDistriMonitor(_agentMgr, _hostDao, _configDao), true, false, false); + _itMgr.registerGuru(VirtualMachine.Type.DomainRouter, this); + + boolean useLocalStorage = Boolean.parseBoolean(configs.get(Config.SystemVMUseLocalStorage.key())); + _offering = new ServiceOfferingVO("System Offering For Software Router", 1, _routerRamSize, _routerCpuMHz, null, + null, true, null, useLocalStorage, true, null, true, VirtualMachine.Type.DomainRouter, true); + _offering.setUniqueName(ServiceOffering.routerDefaultOffUniqueName); + _offering = _serviceOfferingDao.persistSystemServiceOffering(_offering); + + // this can sometimes happen, if DB is manually or programmatically manipulated + if (_offering == null) { + String msg = "Data integrity problem : System Offering For Software router VM has been removed?"; + s_logger.error(msg); + throw new ConfigurationException(msg); + } + + _systemAcct = _accountMgr.getSystemAccount(); + + String aggregationRange = configs.get("usage.stats.job.aggregation.range"); + _usageAggregationRange = NumbersUtil.parseInt(aggregationRange, 1440); + _usageTimeZone = configs.get("usage.aggregation.timezone"); + if (_usageTimeZone == null) { + _usageTimeZone = "GMT"; + } + + _agentMgr.registerForHostEvents(this, true, false, false); + + s_logger.info("DomainRouterManager is configured."); + + return true; + } + + @Override + public String getName() { + return _name; + } + + @Override + public boolean start() { + if (_routerStatsInterval > 0) { + _executor.scheduleAtFixedRate(new NetworkUsageTask(), _routerStatsInterval, _routerStatsInterval, TimeUnit.SECONDS); + } else { + s_logger.debug("router.stats.interval - " + _routerStatsInterval + " so not scheduling the router stats thread"); + } + + // Schedule Network stats update task + TimeZone usageTimezone = TimeZone.getTimeZone(_usageTimeZone); + Calendar cal = Calendar.getInstance(usageTimezone); + cal.setTime(new Date()); + long endDate = 0; + int HOURLY_TIME = 60; + final int DAILY_TIME = 60 * 24; + if (_usageAggregationRange == DAILY_TIME) { + cal.roll(Calendar.DAY_OF_YEAR, false); + cal.set(Calendar.HOUR_OF_DAY, 0); + cal.set(Calendar.MINUTE, 0); + cal.set(Calendar.SECOND, 0); + cal.set(Calendar.MILLISECOND, 0); + cal.roll(Calendar.DAY_OF_YEAR, true); + cal.add(Calendar.MILLISECOND, -1); + endDate = cal.getTime().getTime(); + } else if (_usageAggregationRange == HOURLY_TIME) { + cal.roll(Calendar.HOUR_OF_DAY, false); + cal.set(Calendar.MINUTE, 0); + cal.set(Calendar.SECOND, 0); + cal.set(Calendar.MILLISECOND, 0); + cal.roll(Calendar.HOUR_OF_DAY, true); + cal.add(Calendar.MILLISECOND, -1); + endDate = cal.getTime().getTime(); + } else { + endDate = cal.getTime().getTime(); + } + + _networkStatsUpdateExecutor.scheduleAtFixedRate(new NetworkStatsUpdateTask(), (endDate - System.currentTimeMillis()), + (_usageAggregationRange * 60 * 1000), TimeUnit.MILLISECONDS); + + if (_routerCheckInterval > 0) { + _checkExecutor.scheduleAtFixedRate(new CheckRouterTask(), _routerCheckInterval, _routerCheckInterval, TimeUnit.SECONDS); + } else { + s_logger.debug("router.check.interval - " + _routerCheckInterval + " so not scheduling the redundant router checking thread"); + } + + return true; + } + + @Override + public boolean stop() { + return true; + } + + protected VirtualNetworkApplianceManagerImpl() { + } + + @Override + public Long convertToId(final String vmName) { + if (!VirtualMachineName.isValidRouterName(vmName, _instance)) { + return null; + } + + return VirtualMachineName.getRouterId(vmName); + } + + private VmDataCommand generateVmDataCommand(VirtualRouter router, String vmPrivateIpAddress, String userData, + String serviceOffering, String zoneName, String guestIpAddress, String vmName, + String vmInstanceName, long vmId, String publicKey, long guestNetworkId) { + VmDataCommand cmd = new VmDataCommand(vmPrivateIpAddress, vmName); + + cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); + cmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(guestNetworkId, router.getId())); + cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); + + DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); + cmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); + + cmd.addVmData("userdata", "user-data", userData); + cmd.addVmData("metadata", "service-offering", StringUtils.unicodeEscape(serviceOffering)); + cmd.addVmData("metadata", "availability-zone", StringUtils.unicodeEscape(zoneName)); + cmd.addVmData("metadata", "local-ipv4", guestIpAddress); + cmd.addVmData("metadata", "local-hostname", StringUtils.unicodeEscape(vmName)); + if (dcVo.getNetworkType() == NetworkType.Basic) { + cmd.addVmData("metadata", "public-ipv4", guestIpAddress); + cmd.addVmData("metadata", "public-hostname", StringUtils.unicodeEscape(vmName)); + } else + { + if (router.getPublicIpAddress() == null) { + cmd.addVmData("metadata", "public-ipv4", guestIpAddress); + } else { + cmd.addVmData("metadata", "public-ipv4", router.getPublicIpAddress()); + } + cmd.addVmData("metadata", "public-hostname", router.getPublicIpAddress()); + } + cmd.addVmData("metadata", "instance-id", vmInstanceName); + cmd.addVmData("metadata", "vm-id", String.valueOf(vmId)); + cmd.addVmData("metadata", "public-keys", publicKey); + + String cloudIdentifier = _configDao.getValue("cloud.identifier"); + if (cloudIdentifier == null) { + cloudIdentifier = ""; + } else { + cloudIdentifier = "CloudStack-{" + cloudIdentifier + "}"; + } + cmd.addVmData("metadata", "cloud-identifier", cloudIdentifier); + + return cmd; + } + + protected class NetworkUsageTask implements Runnable { + + public NetworkUsageTask() { + } + + @Override + public void run() { + try { + final List routers = _routerDao.listByStateAndNetworkType(State.Running, GuestType.Isolated, mgmtSrvrId); + s_logger.debug("Found " + routers.size() + " running routers. "); + + for (DomainRouterVO router : routers) { + String privateIP = router.getPrivateIpAddress(); + + if (privateIP != null) { + boolean forVpc = router.getVpcId() != null; + List routerNics = _nicDao.listByVmId(router.getId()); + for (Nic routerNic : routerNics) { + Network network = _networkMgr.getNetwork(routerNic.getNetworkId()); + if ((forVpc && network.getTrafficType() == TrafficType.Public) || (!forVpc && network.getTrafficType() == TrafficType.Guest)) { + final NetworkUsageCommand usageCmd = new NetworkUsageCommand(privateIP, router.getHostName(), + forVpc, routerNic.getIp4Address()); + UserStatisticsVO previousStats = _statsDao.findBy(router.getAccountId(), + router.getDataCenterIdToDeployIn(), network.getId(), (forVpc ? routerNic.getIp4Address() : null), router.getId(), router.getType().toString()); + NetworkUsageAnswer answer = null; + try { + answer = (NetworkUsageAnswer) _agentMgr.easySend(router.getHostId(), usageCmd); + } catch (Exception e) { + s_logger.warn("Error while collecting network stats from router: " + router.getInstanceName() + " from host: " + router.getHostId(), e); + continue; + } + + if (answer != null) { + if (!answer.getResult()) { + s_logger.warn("Error while collecting network stats from router: " + router.getInstanceName() + " from host: " + router.getHostId() + "; details: " + answer.getDetails()); + continue; + } + Transaction txn = Transaction.open(Transaction.CLOUD_DB); + try { + if ((answer.getBytesReceived() == 0) && (answer.getBytesSent() == 0)) { + s_logger.debug("Recieved and Sent bytes are both 0. Not updating user_statistics"); + continue; + } + txn.start(); + UserStatisticsVO stats = _statsDao.lock(router.getAccountId(), + router.getDataCenterIdToDeployIn(), network.getId(), (forVpc ? routerNic.getIp4Address() : null), router.getId(), router.getType().toString()); + if (stats == null) { + s_logger.warn("unable to find stats for account: " + router.getAccountId()); + continue; + } + + if (previousStats != null + && ((previousStats.getCurrentBytesReceived() != stats.getCurrentBytesReceived()) + || (previousStats.getCurrentBytesSent() != stats.getCurrentBytesSent()))) { + s_logger.debug("Router stats changed from the time NetworkUsageCommand was sent. " + + "Ignoring current answer. Router: " + answer.getRouterName() + " Rcvd: " + + answer.getBytesReceived() + "Sent: " + answer.getBytesSent()); + continue; + } + + if (stats.getCurrentBytesReceived() > answer.getBytesReceived()) { + if (s_logger.isDebugEnabled()) { + s_logger.debug("Received # of bytes that's less than the last one. " + + "Assuming something went wrong and persisting it. Router: " + + answer.getRouterName() + " Reported: " + answer.getBytesReceived() + + " Stored: " + stats.getCurrentBytesReceived()); + } + stats.setNetBytesReceived(stats.getNetBytesReceived() + stats.getCurrentBytesReceived()); + } + stats.setCurrentBytesReceived(answer.getBytesReceived()); + if (stats.getCurrentBytesSent() > answer.getBytesSent()) { + if (s_logger.isDebugEnabled()) { + s_logger.debug("Received # of bytes that's less than the last one. " + + "Assuming something went wrong and persisting it. Router: " + + answer.getRouterName() + " Reported: " + answer.getBytesSent() + + " Stored: " + stats.getCurrentBytesSent()); + } + stats.setNetBytesSent(stats.getNetBytesSent() + stats.getCurrentBytesSent()); + } + stats.setCurrentBytesSent(answer.getBytesSent()); + _statsDao.update(stats.getId(), stats); + txn.commit(); + } catch (Exception e) { + txn.rollback(); + s_logger.warn("Unable to update user statistics for account: " + router.getAccountId() + + " Rx: " + answer.getBytesReceived() + "; Tx: " + answer.getBytesSent()); + } finally { + txn.close(); + } + } + if (forVpc) { + // Get VPN gateway + Site2SiteVpnGatewayVO s2sVpn = _s2sVpnGatewayDao.findByVpcId(router.getVpcId()); + if (s2sVpn != null) { + final NetworkUsageCommand vpnUsageCmd = new NetworkUsageCommand(privateIP, router.getHostName(), "vpn", forVpc, routerNic.getIp4Address()); + previousStats = _statsDao.findBy(s2sVpn.getAccountId(), router.getDataCenterIdToDeployIn(), network.getId(), + routerNic.getIp4Address(), s2sVpn.getId(), "VPNGateway"); + answer = null; + try { + answer = (NetworkUsageAnswer) _agentMgr.easySend(router.getHostId(), vpnUsageCmd); + } catch (Exception e) { + s_logger.warn("Error while collecting vpn network stats from router: " + router.getInstanceName() + " from host: " + router.getHostId(), e); + continue; + } + + if (answer != null) { + if (!answer.getResult()) { + s_logger.warn("Error while collecting vpn network stats from router: " + router.getInstanceName() + " from host: " + router.getHostId() + "; details: " + + answer.getDetails()); + continue; + } + Transaction txn = Transaction.open(Transaction.CLOUD_DB); + try { + if ((answer.getBytesReceived() == 0) && (answer.getBytesSent() == 0)) { + s_logger.debug("Recieved and Sent bytes are both 0. Not updating user_statistics"); + continue; + } + txn.start(); + UserStatisticsVO stats = _statsDao.lock(s2sVpn.getAccountId(), router.getDataCenterIdToDeployIn(), network.getId(), + routerNic.getIp4Address(), s2sVpn.getId(), "VPNGateway"); + if (stats == null) { + s_logger.warn("unable to find vpn stats for account: " + router.getAccountId() + " vpc Id: " + router.getVpcId()); + continue; + } + + if (previousStats != null + && ((previousStats.getCurrentBytesReceived() != stats.getCurrentBytesReceived()) + || (previousStats.getCurrentBytesSent() != stats.getCurrentBytesSent()))) { + s_logger.debug("Router stats changed from the time NetworkUsageCommand was sent. " + + "Ignoring current answer. Router: " + answer.getRouterName() + " Rcvd: " + + answer.getBytesReceived() + "Sent: " + answer.getBytesSent()); + continue; + } + + if (stats.getCurrentBytesReceived() > answer.getBytesReceived()) { + if (s_logger.isDebugEnabled()) { + s_logger.debug("Received # of bytes that's less than the last one. " + + "Assuming something went wrong and persisting it. Router: " + + answer.getRouterName() + " Reported: " + answer.getBytesReceived() + + " Stored: " + stats.getCurrentBytesReceived()); + } + stats.setNetBytesReceived(stats.getNetBytesReceived() + stats.getCurrentBytesReceived()); + } + stats.setCurrentBytesReceived(answer.getBytesReceived()); + if (stats.getCurrentBytesSent() > answer.getBytesSent()) { + if (s_logger.isDebugEnabled()) { + s_logger.debug("Received # of bytes that's less than the last one. " + + "Assuming something went wrong and persisting it. Router: " + + answer.getRouterName() + " Reported: " + answer.getBytesSent() + + " Stored: " + stats.getCurrentBytesSent()); + } + stats.setNetBytesSent(stats.getNetBytesSent() + stats.getCurrentBytesSent()); + } + stats.setCurrentBytesSent(answer.getBytesSent()); + _statsDao.update(stats.getId(), stats); + txn.commit(); + } catch (Exception e) { + txn.rollback(); + s_logger.warn("Unable to update user statistics for account: " + router.getAccountId() + + " Rx: " + answer.getBytesReceived() + "; Tx: " + answer.getBytesSent()); + } finally { + txn.close(); + } + } + } + } + } + } + } + } + } catch (Exception e) { + s_logger.warn("Error while collecting network stats", e); + } + } + } + + protected class NetworkStatsUpdateTask implements Runnable { + + public NetworkStatsUpdateTask() { + } + + @Override + public void run() { + GlobalLock scanLock = GlobalLock.getInternLock("network.stats"); + try { + if (scanLock.lock(ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_COOPERATION)) { + // Check for ownership + // msHost in UP state with min id should run the job + ManagementServerHostVO msHost = _msHostDao.findOneInUpState(new Filter(ManagementServerHostVO.class, "id", true, 0L, 1L)); + if (msHost == null || (msHost.getMsid() != mgmtSrvrId)) { + s_logger.debug("Skipping aggregate network stats update"); + scanLock.unlock(); + return; + } + Transaction txn = Transaction.open(Transaction.CLOUD_DB); + try { + txn.start(); + // get all stats with delta > 0 + List updatedStats = _statsDao.listUpdatedStats(); + Date updatedTime = new Date(); + for (UserStatisticsVO stat : updatedStats) { + // update agg bytes + stat.setAggBytesReceived(stat.getCurrentBytesReceived() + stat.getNetBytesReceived()); + stat.setAggBytesSent(stat.getCurrentBytesSent() + stat.getNetBytesSent()); + _userStatsDao.update(stat.getId(), stat); + // insert into op_user_stats_log + UserStatsLogVO statsLog = new UserStatsLogVO(stat.getId(), stat.getNetBytesReceived(), stat.getNetBytesSent(), stat.getCurrentBytesReceived(), + stat.getCurrentBytesSent(), stat.getAggBytesReceived(), stat.getAggBytesSent(), updatedTime); + _userStatsLogDao.persist(statsLog); + } + s_logger.debug("Successfully updated aggregate network stats"); + txn.commit(); + } catch (Exception e) { + txn.rollback(); + s_logger.debug("Failed to update aggregate network stats", e); + } finally { + scanLock.unlock(); + txn.close(); + } + } + } catch (Exception e) { + s_logger.debug("Exception while trying to acquire network stats lock", e); + } finally { + scanLock.releaseRef(); + } + } + } + + @DB + protected void updateSite2SiteVpnConnectionState(List routers) { + for (DomainRouterVO router : routers) { + List conns = _s2sVpnMgr.getConnectionsForRouter(router); + if (conns == null || conns.isEmpty()) { + continue; + } + if (router.getState() != State.Running) { + for (Site2SiteVpnConnectionVO conn : conns) { + if (conn.getState() != Site2SiteVpnConnection.State.Error) { + conn.setState(Site2SiteVpnConnection.State.Disconnected); + _s2sVpnConnectionDao.persist(conn); + } + } + continue; + } + List ipList = new ArrayList(); + for (Site2SiteVpnConnectionVO conn : conns) { + if (conn.getState() != Site2SiteVpnConnection.State.Connected && + conn.getState() != Site2SiteVpnConnection.State.Disconnected) { + continue; + } + Site2SiteCustomerGateway gw = _s2sCustomerGatewayDao.findById(conn.getCustomerGatewayId()); + ipList.add(gw.getGatewayIp()); + } + String privateIP = router.getPrivateIpAddress(); + HostVO host = _hostDao.findById(router.getHostId()); + if (host == null || host.getStatus() != Status.Up) { + continue; + } else if (host.getManagementServerId() != ManagementServerNode.getManagementServerId()) { + /* Only cover hosts managed by this management server */ + continue; + } else if (privateIP != null) { + final CheckS2SVpnConnectionsCommand command = new CheckS2SVpnConnectionsCommand(ipList); + command.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); + command.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); + command.setWait(30); + final Answer origAnswer = _agentMgr.easySend(router.getHostId(), command); + CheckS2SVpnConnectionsAnswer answer = null; + if (origAnswer instanceof CheckS2SVpnConnectionsAnswer) { + answer = (CheckS2SVpnConnectionsAnswer) origAnswer; + } else { + s_logger.warn("Unable to update router " + router.getHostName() + "'s VPN connection status"); + continue; + } + if (!answer.getResult()) { + s_logger.warn("Unable to update router " + router.getHostName() + "'s VPN connection status"); + continue; + } + for (Site2SiteVpnConnectionVO conn : conns) { Site2SiteVpnConnectionVO lock = _s2sVpnConnectionDao.acquireInLockTable(conn.getId()); if (lock == null) { throw new CloudRuntimeException("Unable to acquire lock on " + lock); - } + } try { - if (conn.getState() != Site2SiteVpnConnection.State.Connected && - conn.getState() != Site2SiteVpnConnection.State.Disconnected) { - continue; - } - Site2SiteVpnConnection.State oldState = conn.getState(); - Site2SiteCustomerGateway gw = _s2sCustomerGatewayDao.findById(conn.getCustomerGatewayId()); - if (answer.isConnected(gw.getGatewayIp())) { - conn.setState(Site2SiteVpnConnection.State.Connected); - } else { - conn.setState(Site2SiteVpnConnection.State.Disconnected); - } - _s2sVpnConnectionDao.persist(conn); - if (oldState != conn.getState()) { - String title = "Site-to-site Vpn Connection to " + gw.getName() + - " just switch from " + oldState + " to " + conn.getState(); - String context = "Site-to-site Vpn Connection to " + gw.getName() + " on router " + router.getHostName() + - "(id: " + router.getId() + ") " + " just switch from " + oldState + " to " + conn.getState(); - s_logger.info(context); - _alertMgr.sendAlert(AlertManager.ALERT_TYPE_DOMAIN_ROUTER, - router.getDataCenterIdToDeployIn(), router.getPodIdToDeployIn(), title, context); - } - } finally { - _s2sVpnConnectionDao.releaseFromLockTable(lock.getId()); - } - } - } - } - } - - protected void updateRoutersRedundantState(List routers) { - boolean updated = false; - for (DomainRouterVO router : routers) { - updated = false; - if (!router.getIsRedundantRouter()) { - continue; - } - RedundantState prevState = router.getRedundantState(); - if (router.getState() != State.Running) { - router.setRedundantState(RedundantState.UNKNOWN); - router.setIsPriorityBumpUp(false); - updated = true; - } else { - String privateIP = router.getPrivateIpAddress(); - HostVO host = _hostDao.findById(router.getHostId()); - if (host == null || host.getStatus() != Status.Up) { - router.setRedundantState(RedundantState.UNKNOWN); - updated = true; - } else if (host.getManagementServerId() != ManagementServerNode.getManagementServerId()) { - /* Only cover hosts managed by this management server */ - continue; - } else if (privateIP != null) { - final CheckRouterCommand command = new CheckRouterCommand(); - command.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); - command.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); - command.setWait(60); - final Answer origAnswer = _agentMgr.easySend(router.getHostId(), command); - CheckRouterAnswer answer = null; - if (origAnswer instanceof CheckRouterAnswer) { - answer = (CheckRouterAnswer)origAnswer; - } else { - s_logger.warn("Unable to update router " + router.getHostName() + "'s status"); - } - RedundantState state = RedundantState.UNKNOWN; - boolean isBumped = router.getIsPriorityBumpUp(); - if (answer != null && answer.getResult()) { - state = answer.getState(); - isBumped = answer.isBumped(); - } - router.setRedundantState(state); - router.setIsPriorityBumpUp(isBumped); - updated = true; - } - } - if (updated) { - Transaction txn = Transaction.open(Transaction.CLOUD_DB); - try { - txn.start(); - _routerDao.update(router.getId(), router); - txn.commit(); - } catch (Exception e) { - txn.rollback(); - s_logger.warn("Unable to update router status for account: " + router.getAccountId()); - } finally { - txn.close(); - } - } - RedundantState currState = router.getRedundantState(); - if (prevState != currState) { - String title = "Redundant virtual router " + router.getInstanceName() + - " just switch from " + prevState + " to " + currState; - String context = "Redundant virtual router (name: " + router.getHostName() + ", id: " + router.getId() + ") " + - " just switch from " + prevState + " to " + currState; - s_logger.info(context); - if (currState == RedundantState.MASTER) { - _alertMgr.sendAlert(AlertManager.ALERT_TYPE_DOMAIN_ROUTER, - router.getDataCenterIdToDeployIn(), router.getPodIdToDeployIn(), title, context); - } - } - } - } - - //Ensure router status is update to date before execute this function. The function would try best to recover all routers except MASTER - protected void recoverRedundantNetwork(DomainRouterVO masterRouter, DomainRouterVO backupRouter) { - UserContext context = UserContext.current(); - context.setAccountId(1); - if (masterRouter.getState() == State.Running && backupRouter.getState() == State.Running) { - HostVO masterHost = _hostDao.findById(masterRouter.getHostId()); - HostVO backupHost = _hostDao.findById(backupRouter.getHostId()); - if (masterHost.getStatus() == Status.Up && backupHost.getStatus() == Status.Up) { - String title = "Reboot " + backupRouter.getInstanceName() + " to ensure redundant virtual routers work"; - if (s_logger.isDebugEnabled()) { - s_logger.debug(title); - } - _alertMgr.sendAlert(AlertManager.ALERT_TYPE_DOMAIN_ROUTER, - backupRouter.getDataCenterIdToDeployIn(), backupRouter.getPodIdToDeployIn(), title, title); - try { - rebootRouter(backupRouter.getId(), false); - } catch (ConcurrentOperationException e) { - s_logger.warn("Fail to reboot " + backupRouter.getInstanceName(), e); - } catch (ResourceUnavailableException e) { - s_logger.warn("Fail to reboot " + backupRouter.getInstanceName(), e); - } catch (InsufficientCapacityException e) { - s_logger.warn("Fail to reboot " + backupRouter.getInstanceName(), e); - } - } - } - } - - private int getRealPriority(DomainRouterVO router) { - int priority = router.getPriority(); - if (router.getIsPriorityBumpUp()) { - priority += DEFAULT_DELTA; - } - return priority; - } - - protected class CheckRouterTask implements Runnable { - - public CheckRouterTask() { - } - - /* - * In order to make fail-over works well at any time, we have to ensure: - * 1. Backup router's priority = Master's priority - DELTA + 1 - * 2. Backup router's priority hasn't been bumped up. - */ - private void checkSanity(List routers) { - Set checkedNetwork = new HashSet(); - for (DomainRouterVO router : routers) { - if (!router.getIsRedundantRouter()) { - continue; - } - - List routerGuestNtwkIds = _routerDao.getRouterNetworks(router.getId()); - - for (Long routerGuestNtwkId : routerGuestNtwkIds) { - if (checkedNetwork.contains(routerGuestNtwkId)) { - continue; - } - checkedNetwork.add(routerGuestNtwkId); - List checkingRouters = _routerDao.listByNetworkAndRole(routerGuestNtwkId, Role.VIRTUAL_ROUTER); - if (checkingRouters.size() != 2) { - continue; - } - DomainRouterVO masterRouter = null; - DomainRouterVO backupRouter = null; - for (DomainRouterVO r : checkingRouters) { - if (r.getRedundantState() == RedundantState.MASTER) { - if (masterRouter == null) { - masterRouter = r; - } else { - //Duplicate master! We give up, until the admin fix duplicate MASTER issue - break; - } - } else if (r.getRedundantState() == RedundantState.BACKUP) { - if (backupRouter == null) { - backupRouter = r; - } else { - break; - } - } - } - if (masterRouter != null && backupRouter != null) { - if (getRealPriority(masterRouter) - DEFAULT_DELTA + 1 != getRealPriority(backupRouter) || backupRouter.getIsPriorityBumpUp()) { - recoverRedundantNetwork(masterRouter, backupRouter); - } - } - } - } - } - - private void checkDuplicateMaster(List routers) { - Map networkRouterMaps = new HashMap(); - for (DomainRouterVO router : routers) { - List routerGuestNtwkIds = _routerDao.getRouterNetworks(router.getId()); - - for (Long routerGuestNtwkId : routerGuestNtwkIds) { - if (router.getRedundantState() == RedundantState.MASTER) { - if (networkRouterMaps.containsKey(routerGuestNtwkId)) { - DomainRouterVO dupRouter = networkRouterMaps.get(routerGuestNtwkId); - String title = "More than one redundant virtual router is in MASTER state! Router " + router.getHostName() + " and router " + dupRouter.getHostName(); - String context = "Virtual router (name: " + router.getHostName() + ", id: " + router.getId() + " and router (name: " - + dupRouter.getHostName() + ", id: " + router.getId() + ") are both in MASTER state! If the problem persist, restart both of routers. "; - - _alertMgr.sendAlert(AlertManager.ALERT_TYPE_DOMAIN_ROUTER, router.getDataCenterIdToDeployIn(), router.getPodIdToDeployIn(), title, context); - _alertMgr.sendAlert(AlertManager.ALERT_TYPE_DOMAIN_ROUTER, dupRouter.getDataCenterIdToDeployIn(), dupRouter.getPodIdToDeployIn(), title, context); - } else { - networkRouterMaps.put(routerGuestNtwkId, router); - } - } - } - } - } - - @Override - public void run() { - try { - final List routers = _routerDao.listIsolatedByHostId(null); - s_logger.debug("Found " + routers.size() + " routers. "); - - updateRoutersRedundantState(routers); - updateSite2SiteVpnConnectionState(routers); - - /* FIXME assumed the a pair of redundant routers managed by same mgmt server, - * then the update above can get the latest status */ - checkDuplicateMaster(routers); - checkSanity(routers); - } catch (Exception ex) { - s_logger.error("Fail to complete the CheckRouterTask! ", ex); - } - } - } - - - private final int DEFAULT_PRIORITY = 100; - private final int DEFAULT_DELTA = 2; - - protected int getUpdatedPriority(Network guestNetwork, List routers, DomainRouterVO exclude) throws InsufficientVirtualNetworkCapcityException { - int priority; - if (routers.size() == 0) { - priority = DEFAULT_PRIORITY; - } else { - int maxPriority = 0; - for (DomainRouterVO r : routers) { - if (!r.getIsRedundantRouter()) { - throw new CloudRuntimeException("Redundant router is mixed with single router in one network!"); - } - //FIXME Assume the maxPriority one should be running or just created. - if (r.getId() != exclude.getId() && getRealPriority(r) > maxPriority) { - maxPriority = getRealPriority(r); - } - } - if (maxPriority == 0) { - return DEFAULT_PRIORITY; - } - if (maxPriority < 20) { - s_logger.error("Current maximum priority is too low!"); - throw new InsufficientVirtualNetworkCapcityException("Current maximum priority is too low as " + maxPriority + "!", - guestNetwork.getId()); - } else if (maxPriority > 200) { - s_logger.error("Too many times fail-over happened! Current maximum priority is too high as " + maxPriority + "!"); - throw new InsufficientVirtualNetworkCapcityException("Too many times fail-over happened! Current maximum priority is too high as " - + maxPriority + "!", guestNetwork.getId()); - } - priority = maxPriority - DEFAULT_DELTA + 1; - } - return priority; - } - - /* - * Ovm won't support any system. So we have to choose a partner cluster in the same pod to start domain router for us - */ - private HypervisorType getClusterToStartDomainRouterForOvm(long podId) { - List clusters = _clusterDao.listByPodId(podId); - for (ClusterVO cv : clusters) { - if (cv.getHypervisorType() == HypervisorType.Ovm || cv.getHypervisorType() == HypervisorType.BareMetal) { - continue; - } - - List hosts = _resourceMgr.listAllHostsInCluster(cv.getId()); - if (hosts == null || hosts.isEmpty()) { - continue; - } - - for (HostVO h : hosts) { - if (h.getStatus() == Status.Up) { - s_logger.debug("Pick up host that has hypervisor type " + h.getHypervisorType() + " in cluster " + - cv.getId() + " to start domain router for OVM"); - return h.getHypervisorType(); - } - } - } - - String errMsg = "Cannot find an available cluster in Pod " - + podId - + " to start domain router for Ovm. \n Ovm won't support any system vm including domain router, " + - "please make sure you have a cluster with hypervisor type of any of xenserver/KVM/Vmware in the same pod" + - " with Ovm cluster. And there is at least one host in UP status in that cluster."; - throw new CloudRuntimeException(errMsg); - } - - @DB - protected List findOrDeployVirtualRouterInGuestNetwork(Network guestNetwork, DeployDestination dest, Account owner, - boolean isRedundant, Map params) throws ConcurrentOperationException, - InsufficientCapacityException, ResourceUnavailableException { - - assert guestNetwork.getState() == Network.State.Implemented || guestNetwork.getState() == Network.State.Setup || - guestNetwork.getState() == Network.State.Implementing : "Network is not yet fully implemented: " - + guestNetwork; - assert guestNetwork.getTrafficType() == TrafficType.Guest; - - - - //1) Get deployment plan and find out the list of routers - boolean isPodBased = (dest.getDataCenter().getNetworkType() == NetworkType.Basic || - _networkMgr.areServicesSupportedInNetwork(guestNetwork.getId(), Service.SecurityGroup)) - && guestNetwork.getTrafficType() == TrafficType.Guest; - - Pair> planAndRouters = getDeploymentPlanAndRouters(isPodBased, dest, guestNetwork.getId()); - DeploymentPlan plan = planAndRouters.first(); - List routers = planAndRouters.second(); - - //2) Figure out required routers count - int routerCount = 1; - if (isRedundant) { - routerCount = 2; - } - - /* If it is the single router network, then keep it untouched */ - for (DomainRouterVO router : routers) { - if (!router.getIsRedundantRouter() || isPodBased) { - routerCount = 1; - break; - } - } - - /* If old network is redundant but new is single router, then routers.size() = 2 but routerCount = 1 */ - if (routers.size() >= routerCount) { - return routers; - } - - if (routers.size() >= 5) { - s_logger.error("Too much redundant routers!"); - } - - Network network = _networkDao.acquireInLockTable(guestNetwork.getId()); - if (network == null) { - throw new ConcurrentOperationException("Unable to lock network " + guestNetwork.getId()); - } - - try { - //Check if providers are supported in the physical networks - VirtualRouterProviderType type = VirtualRouterProviderType.VirtualRouter; - Long physicalNetworkId = _networkMgr.getPhysicalNetworkId(network); - PhysicalNetworkServiceProvider provider = _physicalProviderDao.findByServiceProvider(physicalNetworkId, type.toString()); - if (provider == null) { - throw new CloudRuntimeException("Cannot find service provider " + type.toString() + " in physical network " + physicalNetworkId); - } - VirtualRouterProvider vrProvider = _vrProviderDao.findByNspIdAndType(provider.getId(), type); - if (vrProvider == null) { - throw new CloudRuntimeException("Cannot find virtual router provider " + type.toString()+ " as service provider " + provider.getId()); - } - - if (_networkMgr.isNetworkSystem(guestNetwork) || guestNetwork.getGuestType() == Network.GuestType.Shared) { - owner = _accountMgr.getAccount(Account.ACCOUNT_ID_SYSTEM); - } - - //Check if public network has to be set on VR - boolean publicNetwork = false; - if (_networkMgr.isProviderSupportServiceInNetwork(guestNetwork.getId(), Service.SourceNat, Provider.VirtualRouter)) { - publicNetwork = true; - } - if (isRedundant && !publicNetwork) { - s_logger.error("Didn't support redundant virtual router without public network!"); - return null; - } - - Long offeringId = _networkOfferingDao.findById(guestNetwork.getNetworkOfferingId()).getServiceOfferingId(); - if (offeringId == null) { - offeringId = _offering.getId(); - } - - PublicIp sourceNatIp = null; - if (publicNetwork) { - sourceNatIp = _networkMgr.assignSourceNatIpAddressToGuestNetwork(owner, guestNetwork); - } - - //3) deploy virtual router(s) - int count = routerCount - routers.size(); - for (int i = 0; i < count; i++) { - List> networks = createRouterNetworks(owner, isRedundant, plan, guestNetwork, - new Pair(publicNetwork, sourceNatIp)); - DomainRouterVO router = deployRouter(owner, dest, plan, params, isRedundant, vrProvider, offeringId, - null, networks); - - _routerDao.addRouterToGuestNetwork(router, network); - - routers.add(router); - } - } finally { - if (network != null) { - _networkDao.releaseFromLockTable(network.getId()); - } - } - return routers; - } - - protected DomainRouterVO deployRouter(Account owner, DeployDestination dest, DeploymentPlan plan, Map params, - boolean isRedundant, VirtualRouterProvider vrProvider, long svcOffId, - Long vpcId, List> networks) throws ConcurrentOperationException, - InsufficientAddressCapacityException, InsufficientServerCapacityException, InsufficientCapacityException, - StorageUnavailableException, ResourceUnavailableException { - - long id = _routerDao.getNextInSequence(Long.class, "id"); - if (s_logger.isDebugEnabled()) { - s_logger.debug("Creating the router " + id + " in datacenter " + dest.getDataCenter()); - } - - ServiceOfferingVO routerOffering = _serviceOfferingDao.findById(svcOffId); - - // Router is the network element, we don't know the hypervisor type yet. - //Try to allocate the domR twice using diff hypervisors, and when failed both times, throw the exception up - List supportedHypervisors = new ArrayList(); - HypervisorType defaults = _resourceMgr.getDefaultHypervisor(dest.getDataCenter().getId()); - if (defaults != HypervisorType.None) { - supportedHypervisors.add(defaults); - } - - if (dest.getCluster() != null) { - if (dest.getCluster().getHypervisorType() == HypervisorType.Ovm) { - supportedHypervisors.add(getClusterToStartDomainRouterForOvm(dest.getCluster().getPodId())); - } else { - supportedHypervisors.add(dest.getCluster().getHypervisorType()); - } - } else { - supportedHypervisors = _resourceMgr.getSupportedHypervisorTypes(dest.getDataCenter().getId(), true, - plan.getPodId()); - } - - if (supportedHypervisors.isEmpty()) { - if (plan.getPodId() != null) { - throw new InsufficientServerCapacityException("Unable to create virtual router, " + - "there are no clusters in the pod ", Pod.class, plan.getPodId()); - } - throw new InsufficientServerCapacityException("Unable to create virtual router, " + - "there are no clusters in the zone ", DataCenter.class, dest.getDataCenter().getId()); - } - - int allocateRetry = 0; - int startRetry = 0; - DomainRouterVO router = null; - for (Iterator iter = supportedHypervisors.iterator();iter.hasNext();) { - HypervisorType hType = iter.next(); - try { - s_logger.debug("Allocating the domR with the hypervisor type " + hType); - VMTemplateVO template = _templateDao.findRoutingTemplate(hType); - - if (template == null) { - s_logger.debug(hType + " won't support system vm, skip it"); - continue; - } - - boolean offerHA = routerOffering.getOfferHA(); - /* We don't provide HA to redundant router VMs, admin should own it all, and redundant router themselves are HA */ - if (isRedundant) { - offerHA = false; - } - - router = new DomainRouterVO(id, routerOffering.getId(), vrProvider.getId(), - VirtualMachineName.getRouterName(id, _instance), template.getId(), template.getHypervisorType(), - template.getGuestOSId(), owner.getDomainId(), owner.getId(), isRedundant, 0, false, - RedundantState.UNKNOWN, offerHA, false, vpcId); - router.setRole(Role.VIRTUAL_ROUTER); - router = _itMgr.allocate(router, template, routerOffering, networks, plan, null, owner); - } catch (InsufficientCapacityException ex) { - if (allocateRetry < 2 && iter.hasNext()) { - s_logger.debug("Failed to allocate the domR with hypervisor type " + hType + ", retrying one more time"); - continue; - } else { - throw ex; - } - } finally { - allocateRetry++; - } - - try { - router = startVirtualRouter(router, _accountMgr.getSystemUser(), _accountMgr.getSystemAccount(), params); - break; - } catch (InsufficientCapacityException ex) { - if (startRetry < 2 && iter.hasNext()) { - s_logger.debug("Failed to start the domR " + router + " with hypervisor type " + hType + ", " + - "destroying it and recreating one more time"); - //destroy the router - destroyRouter(router.getId()); - continue; - } else { - throw ex; - } - } finally { - startRetry++; - } - } - - return router; - } - - protected List> createRouterNetworks(Account owner, boolean isRedundant, - DeploymentPlan plan, Network guestNetwork, Pair publicNetwork) throws ConcurrentOperationException, - InsufficientAddressCapacityException { - - - boolean setupPublicNetwork = false; - if (publicNetwork != null) { - setupPublicNetwork = publicNetwork.first(); - } - - //Form networks - List> networks = new ArrayList>(3); - - //1) Guest network - boolean hasGuestNetwork = false; - if (guestNetwork != null) { - s_logger.debug("Adding nic for Virtual Router in Guest network " + guestNetwork); - String defaultNetworkStartIp = null; - if (guestNetwork.getCidr() != null && !setupPublicNetwork) { - String startIp = _networkMgr.getStartIpAddress(guestNetwork.getId()); - if (startIp != null && _ipAddressDao.findByIpAndSourceNetworkId(guestNetwork.getId(), startIp).getAllocatedTime() == null) { - defaultNetworkStartIp = startIp; - } else if (s_logger.isDebugEnabled()){ - s_logger.debug("First ip " + startIp + " in network id=" + guestNetwork.getId() + - " is already allocated, can't use it for domain router; will get random ip address from the range"); - } - } - - NicProfile gatewayNic = new NicProfile(defaultNetworkStartIp); - if (setupPublicNetwork) { - if (isRedundant) { - gatewayNic.setIp4Address(_networkMgr.acquireGuestIpAddress(guestNetwork, null)); - } else { - gatewayNic.setIp4Address(guestNetwork.getGateway()); - } - gatewayNic.setBroadcastUri(guestNetwork.getBroadcastUri()); - gatewayNic.setBroadcastType(guestNetwork.getBroadcastDomainType()); - gatewayNic.setIsolationUri(guestNetwork.getBroadcastUri()); - gatewayNic.setMode(guestNetwork.getMode()); - String gatewayCidr = guestNetwork.getCidr(); - gatewayNic.setNetmask(NetUtils.getCidrNetmask(gatewayCidr)); - } else { - gatewayNic.setDefaultNic(true); - } - - networks.add(new Pair((NetworkVO) guestNetwork, gatewayNic)); - hasGuestNetwork = true; - } - - //2) Control network - s_logger.debug("Adding nic for Virtual Router in Control network "); - List offerings = _networkMgr.getSystemAccountNetworkOfferings(NetworkOfferingVO.SystemControlNetwork); - NetworkOfferingVO controlOffering = offerings.get(0); - NetworkVO controlConfig = _networkMgr.setupNetwork(_systemAcct, controlOffering, plan, null, null, false).get(0); - networks.add(new Pair(controlConfig, null)); - - - //3) Public network - if (setupPublicNetwork) { - PublicIp sourceNatIp = publicNetwork.second(); - s_logger.debug("Adding nic for Virtual Router in Public network "); - //if source nat service is supported by the network, get the source nat ip address - NicProfile defaultNic = new NicProfile(); - defaultNic.setDefaultNic(true); - defaultNic.setIp4Address(sourceNatIp.getAddress().addr()); - defaultNic.setGateway(sourceNatIp.getGateway()); - defaultNic.setNetmask(sourceNatIp.getNetmask()); - defaultNic.setMacAddress(sourceNatIp.getMacAddress()); - defaultNic.setBroadcastType(BroadcastDomainType.Vlan); - defaultNic.setBroadcastUri(BroadcastDomainType.Vlan.toUri(sourceNatIp.getVlanTag())); - defaultNic.setIsolationUri(IsolationType.Vlan.toUri(sourceNatIp.getVlanTag())); - if (hasGuestNetwork) { - defaultNic.setDeviceId(2); - } - NetworkOfferingVO publicOffering = _networkMgr.getSystemAccountNetworkOfferings(NetworkOfferingVO.SystemPublicNetwork).get(0); - List publicNetworks = _networkMgr.setupNetwork(_systemAcct, publicOffering, plan, null, null, false); - networks.add(new Pair(publicNetworks.get(0), defaultNic)); - } - - return networks; - } - - - protected Pair> getDeploymentPlanAndRouters(boolean isPodBased, - DeployDestination dest, long guestNetworkId) { - long dcId = dest.getDataCenter().getId(); - List routers = null; - DeploymentPlan plan = new DataCenterDeployment(dcId); - if (isPodBased) { - Pod pod = dest.getPod(); - Long podId = null; - if (pod != null) { - podId = pod.getId(); - } else { - throw new CloudRuntimeException("Pod id is expected in deployment destination"); - } - routers = _routerDao.listByNetworkAndPodAndRole(guestNetworkId, podId, Role.VIRTUAL_ROUTER); - plan = new DataCenterDeployment(dcId, podId, null, null, null, null); - } else { - routers = _routerDao.listByNetworkAndRole(guestNetworkId, Role.VIRTUAL_ROUTER); - } - - return new Pair>(plan, routers); - } - - - private DomainRouterVO startVirtualRouter(DomainRouterVO router, User user, Account caller, Map params) - throws StorageUnavailableException, InsufficientCapacityException, - ConcurrentOperationException, ResourceUnavailableException { - - if (router.getRole() != Role.VIRTUAL_ROUTER || !router.getIsRedundantRouter()) { - return this.start(router, user, caller, params, null); - } - - if (router.getState() == State.Running) { - s_logger.debug("Redundant router " + router.getInstanceName() + " is already running!"); - return router; - } - - DataCenterDeployment plan = new DataCenterDeployment(0, null, null, null, null, null); - DomainRouterVO result = null; - assert router.getIsRedundantRouter(); - List routerList = _routerDao.findBy(router.getAccountId(), router.getDataCenterIdToDeployIn()); - DomainRouterVO routerToBeAvoid = null; - for (DomainRouterVO rrouter : routerList) { - if (rrouter.getHostId() != null && rrouter.getIsRedundantRouter() && rrouter.getState() == State.Running) { - if (routerToBeAvoid != null) { - throw new ResourceUnavailableException("Try to start router " + router.getInstanceName() + "(" + router.getId() + ")" - + ", but there are already two redundant routers with IP " + router.getPublicIpAddress() - + ", they are " + rrouter.getInstanceName() + "(" + rrouter.getId() + ") and " - + routerToBeAvoid.getInstanceName() + "(" + routerToBeAvoid.getId() + ")", - DataCenter.class, rrouter.getDataCenterIdToDeployIn()); - } - routerToBeAvoid = rrouter; - } - } - if (routerToBeAvoid == null) { - return this.start(router, user, caller, params, null); - } - // We would try best to deploy the router to another place - int retryIndex = 5; - ExcludeList[] avoids = new ExcludeList[5]; - avoids[0] = new ExcludeList(); - avoids[0].addPod(routerToBeAvoid.getPodIdToDeployIn()); - avoids[1] = new ExcludeList(); - avoids[1].addCluster(_hostDao.findById(routerToBeAvoid.getHostId()).getClusterId()); - avoids[2] = new ExcludeList(); - List volumes = _volumeDao.findByInstanceAndType(routerToBeAvoid.getId(), Type.ROOT); - if (volumes != null && volumes.size() != 0) { - avoids[2].addPool(volumes.get(0).getPoolId()); - } - avoids[2].addHost(routerToBeAvoid.getHostId()); - avoids[3] = new ExcludeList(); - avoids[3].addHost(routerToBeAvoid.getHostId()); - avoids[4] = new ExcludeList(); - - for (int i = 0; i < retryIndex; i++) { - if (s_logger.isTraceEnabled()) { - s_logger.trace("Try to deploy redundant virtual router:" + router.getHostName() + ", for " + i + " time"); - } - plan.setAvoids(avoids[i]); - try { - result = this.start(router, user, caller, params, plan); - } catch (InsufficientServerCapacityException ex) { - result = null; - } - if (result != null) { - break; - } - } - return result; - } - - @Override - public List deployVirtualRouterInGuestNetwork(Network guestNetwork, DeployDestination dest, Account owner, - Map params, boolean isRedundant) throws InsufficientCapacityException, - ConcurrentOperationException, ResourceUnavailableException { - - List routers = findOrDeployVirtualRouterInGuestNetwork - (guestNetwork, dest, owner, isRedundant, params); - - return startRouters(params, routers); - } - - protected List startRouters(Map params, List routers) throws StorageUnavailableException, InsufficientCapacityException, ConcurrentOperationException, - ResourceUnavailableException { - List runningRouters = null; - - if (routers != null) { - runningRouters = new ArrayList(); - } - - for (DomainRouterVO router : routers) { - boolean skip = false; - State state = router.getState(); - if (router.getHostId() != null && state != State.Running) { - HostVO host = _hostDao.findById(router.getHostId()); - if (host == null || host.getStatus() != Status.Up) { - skip = true; - } - } - if (!skip) { - if (state != State.Running) { - router = startVirtualRouter(router, _accountMgr.getSystemUser(), _accountMgr.getSystemAccount(), params); - } - if (router != null) { - runningRouters.add(router); - } - } - } - return runningRouters; - } - - @Override - public boolean finalizeVirtualMachineProfile(VirtualMachineProfile profile, DeployDestination dest, - ReservationContext context) { - - boolean dnsProvided = true; - boolean dhcpProvided = true; - boolean publicNetwork = false; - DataCenterVO dc = _dcDao.findById(dest.getDataCenter().getId()); - _dcDao.loadDetails(dc); - - //1) Set router details - DomainRouterVO router = profile.getVirtualMachine(); - Map details = _vmDetailsDao.findDetails(router.getId()); - router.setDetails(details); - - //2) Prepare boot loader elements related with Control network - - StringBuilder buf = profile.getBootArgsBuilder(); - buf.append(" template=domP"); - buf.append(" name=").append(profile.getHostName()); - - if (Boolean.valueOf(_configDao.getValue("system.vm.random.password"))) { - buf.append(" vmpassword=").append(_configDao.getValue("system.vm.password")); - } - - NicProfile controlNic = null; - String defaultDns1 = null; - String defaultDns2 = null; - for (NicProfile nic : profile.getNics()) { - int deviceId = nic.getDeviceId(); - buf.append(" eth").append(deviceId).append("ip=").append(nic.getIp4Address()); - buf.append(" eth").append(deviceId).append("mask=").append(nic.getNetmask()); - - if (nic.isDefaultNic()) { - buf.append(" gateway=").append(nic.getGateway()); - defaultDns1 = nic.getDns1(); - defaultDns2 = nic.getDns2(); - } - - if (nic.getTrafficType() == TrafficType.Management) { - buf.append(" localgw=").append(dest.getPod().getGateway()); - } else if (nic.getTrafficType() == TrafficType.Control) { - controlNic = nic; - // DOMR control command is sent over management server in VMware - if (dest.getHost().getHypervisorType() == HypervisorType.VMware) { - if (s_logger.isInfoEnabled()) { - s_logger.info("Check if we need to add management server explicit route to DomR. pod cidr: " - + dest.getPod().getCidrAddress() + "/" + dest.getPod().getCidrSize() - + ", pod gateway: " + dest.getPod().getGateway() + ", management host: " + _mgmt_host); - } - - if (s_logger.isInfoEnabled()) { - s_logger.info("Add management server explicit route to DomR."); - } - - // always add management explicit route, for basic networking setup, DomR may have two interfaces while both - // are on the same subnet - _mgmt_cidr = _configDao.getValue(Config.ManagementNetwork.key()); - if (NetUtils.isValidCIDR(_mgmt_cidr)) { - buf.append(" mgmtcidr=").append(_mgmt_cidr); - buf.append(" localgw=").append(dest.getPod().getGateway()); - } - - - if (dc.getNetworkType() == NetworkType.Basic) { - // ask domR to setup SSH on guest network - buf.append(" sshonguest=true"); - } - - } - } else if (nic.getTrafficType() == TrafficType.Guest) { - dnsProvided = _networkMgr.isProviderSupportServiceInNetwork(nic.getNetworkId(), Service.Dns, Provider.VirtualRouter); - dhcpProvided = _networkMgr.isProviderSupportServiceInNetwork(nic.getNetworkId(), Service.Dhcp, Provider.VirtualRouter); - //build bootloader parameter for the guest - buf.append(createGuestBootLoadArgs(nic, defaultDns1, defaultDns2, router)); - } else if (nic.getTrafficType() == TrafficType.Public) { - publicNetwork = true; - } - } - - if (controlNic == null) { - throw new CloudRuntimeException("Didn't start a control port"); - } - - String rpValue = _configDao.getValue(Config.NetworkRouterRpFilter.key()); - if (rpValue != null && rpValue.equalsIgnoreCase("true")) { - _disable_rp_filter = true; - }else { - _disable_rp_filter = false; - } - - String rpFilter = " "; - String type = null; - if (router.getVpcId() != null) { - type = "vpcrouter"; - if (_disable_rp_filter) { - rpFilter=" disable_rp_filter=true"; - } - } else if (!publicNetwork) { - type = "dhcpsrvr"; - } else { - type = "router"; - if (_disable_rp_filter) { - rpFilter=" disable_rp_filter=true"; - } - } - - if (_disable_rp_filter) { - rpFilter=" disable_rp_filter=true"; - } - - buf.append(" type=" + type + rpFilter); - - String domain_suffix = dc.getDetail(ZoneConfig.DnsSearchOrder.getName()); - if (domain_suffix != null) { - buf.append(" dnssearchorder=").append(domain_suffix); - } - - if (profile.getHypervisorType() == HypervisorType.VMware) { - buf.append(" extra_pubnics=" + _routerExtraPublicNics); - } - - /* If virtual router didn't provide DNS service but provide DHCP service, we need to override the DHCP response - * to return DNS server rather than - * virtual router itself. */ - if (dnsProvided || dhcpProvided) { - if (defaultDns1 != null) { - buf.append(" dns1=").append(defaultDns1); - } - if (defaultDns2 != null) { - buf.append(" dns2=").append(defaultDns2); - } - - boolean useExtDns = !dnsProvided; - /* For backward compatibility */ - String use_external_dns = _configDao.getValue(Config.UseExternalDnsServers.key()); - if (use_external_dns != null && use_external_dns.equals("true")) { - useExtDns = true; - } - - if (useExtDns) { - buf.append(" useextdns=true"); - } - } - - if (s_logger.isDebugEnabled()) { - s_logger.debug("Boot Args for " + profile + ": " + buf.toString()); - } - - return true; - } - - - protected StringBuilder createGuestBootLoadArgs(NicProfile guestNic, String defaultDns1, - String defaultDns2, DomainRouterVO router) { - long guestNetworkId = guestNic.getNetworkId(); - NetworkVO guestNetwork = _networkDao.findById(guestNetworkId); - String dhcpRange = null; - DataCenterVO dc = _dcDao.findById(guestNetwork.getDataCenterId()); - - StringBuilder buf = new StringBuilder(); - - boolean isRedundant = router.getIsRedundantRouter(); - if (isRedundant) { - buf.append(" redundant_router=1"); - List routers = _routerDao.listByNetworkAndRole(guestNetwork.getId(), Role.VIRTUAL_ROUTER); - try { - int priority = getUpdatedPriority(guestNetwork, routers, router); - router.setPriority(priority); - } catch (InsufficientVirtualNetworkCapcityException e) { - s_logger.error("Failed to get update priority!", e); - throw new CloudRuntimeException("Failed to get update priority!"); - } - Network net = _networkMgr.getNetwork(guestNic.getNetworkId()); - buf.append(" guestgw=").append(net.getGateway()); - String brd = NetUtils.long2Ip(NetUtils.ip2Long(guestNic.getIp4Address()) | ~NetUtils.ip2Long(guestNic.getNetmask())); - buf.append(" guestbrd=").append(brd); - buf.append(" guestcidrsize=").append(NetUtils.getCidrSize(guestNic.getNetmask())); - buf.append(" router_pr=").append(router.getPriority()); - } - - //setup network domain - String domain = guestNetwork.getNetworkDomain(); - if (domain != null) { - buf.append(" domain=" + domain); - } - - //setup dhcp range - if (dc.getNetworkType() == NetworkType.Basic) { - if (guestNic.isDefaultNic()) { - long cidrSize = NetUtils.getCidrSize(guestNic.getNetmask()); - String cidr = NetUtils.getCidrSubNet(guestNic.getGateway(), cidrSize); - if (cidr != null) { - dhcpRange = NetUtils.getIpRangeStartIpFromCidr(cidr, cidrSize); - } - } - } else if (dc.getNetworkType() == NetworkType.Advanced) { - String cidr = guestNetwork.getCidr(); - if (cidr != null) { - dhcpRange = NetUtils.getDhcpRange(cidr); - } - } - - if (dhcpRange != null) { - buf.append(" dhcprange=" + dhcpRange); - } - - return buf; - } - - - protected String getGuestDhcpRange(NicProfile guestNic, Network guestNetwork, DataCenter dc) { - String dhcpRange = null; - //setup dhcp range - if (dc.getNetworkType() == NetworkType.Basic) { - long cidrSize = NetUtils.getCidrSize(guestNic.getNetmask()); - String cidr = NetUtils.getCidrSubNet(guestNic.getGateway(), cidrSize); - if (cidr != null) { - dhcpRange = NetUtils.getIpRangeStartIpFromCidr(cidr, cidrSize); - } - } else if (dc.getNetworkType() == NetworkType.Advanced) { - String cidr = guestNetwork.getCidr(); - if (cidr != null) { - dhcpRange = NetUtils.getDhcpRange(cidr); - } - } - return dhcpRange; - } - - @Override - public boolean finalizeDeployment(Commands cmds, VirtualMachineProfile profile, - DeployDestination dest, ReservationContext context) throws ResourceUnavailableException { - DomainRouterVO router = profile.getVirtualMachine(); - - List nics = profile.getNics(); - for (NicProfile nic : nics) { - if (nic.getTrafficType() == TrafficType.Public) { - router.setPublicIpAddress(nic.getIp4Address()); - router.setPublicNetmask(nic.getNetmask()); - router.setPublicMacAddress(nic.getMacAddress()); - } else if (nic.getTrafficType() == TrafficType.Control) { - router.setPrivateIpAddress(nic.getIp4Address()); - router.setPrivateMacAddress(nic.getMacAddress()); - } - } - _routerDao.update(router.getId(), router); - - finalizeCommandsOnStart(cmds, profile); - return true; - } - - @Override - public boolean finalizeCommandsOnStart(Commands cmds, VirtualMachineProfile profile) { - DomainRouterVO router = profile.getVirtualMachine(); - NicProfile controlNic = getControlNic(profile); - - if (controlNic == null) { - s_logger.error("Control network doesn't exist for the router " + router); - return false; - } - - finalizeSshAndVersionAndNetworkUsageOnStart(cmds, profile, router, controlNic); - - // restart network if restartNetwork = false is not specified in profile parameters - boolean reprogramGuestNtwks = true; - if (profile.getParameter(Param.ReProgramGuestNetworks) != null - && (Boolean) profile.getParameter(Param.ReProgramGuestNetworks) == false) { - reprogramGuestNtwks = false; - } - - VirtualRouterProvider vrProvider = _vrProviderDao.findById(router.getElementId()); - if (vrProvider == null) { - throw new CloudRuntimeException("Cannot find related virtual router provider of router: " + router.getHostName()); - } - Provider provider = Network.Provider.getProvider(vrProvider.getType().toString()); - if (provider == null) { - throw new CloudRuntimeException("Cannot find related provider of virtual router provider: " + vrProvider.getType().toString()); - } - - List routerGuestNtwkIds = _routerDao.getRouterNetworks(router.getId()); - for (Long guestNetworkId : routerGuestNtwkIds) { - if (reprogramGuestNtwks) { - finalizeIpAssocForNetwork(cmds, router, provider, guestNetworkId, null); - finalizeNetworkRulesForNetwork(cmds, router, provider, guestNetworkId); - } - - finalizeUserDataAndDhcpOnStart(cmds, router, provider, guestNetworkId); - } - - return true; - } - - protected NicProfile getControlNic(VirtualMachineProfile profile) { - DomainRouterVO router = profile.getVirtualMachine(); - DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); - NicProfile controlNic = null; - if (profile.getHypervisorType() == HypervisorType.VMware && dcVo.getNetworkType() == NetworkType.Basic) { - // TODO this is a ugly to test hypervisor type here - // for basic network mode, we will use the guest NIC for control NIC - for (NicProfile nic : profile.getNics()) { - if (nic.getTrafficType() == TrafficType.Guest && nic.getIp4Address() != null) { - controlNic = nic; - } - } - } else { - for (NicProfile nic : profile.getNics()) { - if (nic.getTrafficType() == TrafficType.Control && nic.getIp4Address() != null) { - controlNic = nic; - } - } - } - return controlNic; - } - - protected void finalizeSshAndVersionAndNetworkUsageOnStart(Commands cmds, VirtualMachineProfile profile, DomainRouterVO router, NicProfile controlNic) { - cmds.addCommand("checkSsh", new CheckSshCommand(profile.getInstanceName(), controlNic.getIp4Address(), 3922)); - - // Update router template/scripts version - final GetDomRVersionCmd command = new GetDomRVersionCmd(); - command.setAccessDetail(NetworkElementCommand.ROUTER_IP, controlNic.getIp4Address()); - command.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); - cmds.addCommand("getDomRVersion", command); - - // Network usage command to create iptables rules - boolean forVpc = profile.getVirtualMachine().getVpcId() != null; - cmds.addCommand("networkUsage", new NetworkUsageCommand(controlNic.getIp4Address(), router.getHostName(), "create", forVpc)); - } - - protected void finalizeUserDataAndDhcpOnStart(Commands cmds, DomainRouterVO router, Provider provider, Long guestNetworkId) { - if (_networkMgr.isProviderSupportServiceInNetwork(guestNetworkId, Service.Dhcp, provider)) { - // Resend dhcp - s_logger.debug("Reapplying dhcp entries as a part of domR " + router + " start..."); - createDhcpEntryCommandsForVMs(router, cmds, guestNetworkId); - } - - if (_networkMgr.isProviderSupportServiceInNetwork(guestNetworkId, Service.UserData, provider)) { - // Resend user data - s_logger.debug("Reapplying vm data (userData and metaData) entries as a part of domR " + router + " start..."); - createVmDataCommandForVMs(router, cmds, guestNetworkId); - } - } - - protected void finalizeNetworkRulesForNetwork(Commands cmds, DomainRouterVO router, Provider provider, Long guestNetworkId) { - s_logger.debug("Resending ipAssoc, port forwarding, load balancing rules as a part of Virtual router start"); - - ArrayList publicIps = getPublicIpsToApply(router, provider, guestNetworkId); - - if (publicIps != null && !publicIps.isEmpty()) { - List vpns = new ArrayList(); - List pfRules = new ArrayList(); - List staticNatFirewallRules = new ArrayList(); - List staticNats = new ArrayList(); - List firewallRules = new ArrayList(); - - //Get information about all the rules (StaticNats and StaticNatRules; PFVPN to reapply on domR start) - for (PublicIpAddress ip : publicIps) { - if (_networkMgr.isProviderSupportServiceInNetwork(guestNetworkId, Service.PortForwarding, provider)) { - pfRules.addAll(_pfRulesDao.listForApplication(ip.getId())); - } - if (_networkMgr.isProviderSupportServiceInNetwork(guestNetworkId, Service.StaticNat, provider)) { - staticNatFirewallRules.addAll(_rulesDao.listByIpAndPurpose(ip.getId(), Purpose.StaticNat)); - } - if (_networkMgr.isProviderSupportServiceInNetwork(guestNetworkId, Service.Firewall, provider)) { - firewallRules.addAll(_rulesDao.listByIpAndPurpose(ip.getId(), Purpose.Firewall)); - } - - if (_networkMgr.isProviderSupportServiceInNetwork(guestNetworkId, Service.Vpn, provider)) { - RemoteAccessVpn vpn = _vpnDao.findById(ip.getId()); - if (vpn != null) { - vpns.add(vpn); - } - } - - if (_networkMgr.isProviderSupportServiceInNetwork(guestNetworkId, Service.StaticNat, provider)) { - if (ip.isOneToOneNat()) { - String dstIp = _networkMgr.getIpInNetwork(ip.getAssociatedWithVmId(), guestNetworkId); - StaticNatImpl staticNat = new StaticNatImpl(ip.getAccountId(), ip.getDomainId(), guestNetworkId, ip.getId(), dstIp, false); - staticNats.add(staticNat); - } - } - } - - //Re-apply static nats - s_logger.debug("Found " + staticNats.size() + " static nat(s) to apply as a part of domR " + router + " start."); - if (!staticNats.isEmpty()) { - createApplyStaticNatCommands(staticNats, router, cmds, guestNetworkId); - } - - //Re-apply firewall rules - s_logger.debug("Found " + staticNats.size() + " firewall rule(s) to apply as a part of domR " + router + " start."); - if (!firewallRules.isEmpty()) { - createFirewallRulesCommands(firewallRules, router, cmds, guestNetworkId); - } - - // Re-apply port forwarding rules - s_logger.debug("Found " + pfRules.size() + " port forwarding rule(s) to apply as a part of domR " + router + " start."); - if (!pfRules.isEmpty()) { - createApplyPortForwardingRulesCommands(pfRules, router, cmds, guestNetworkId); - } - - // Re-apply static nat rules - s_logger.debug("Found " + staticNatFirewallRules.size() + " static nat rule(s) to apply as a part of domR " + router + " start."); - if (!staticNatFirewallRules.isEmpty()) { - List staticNatRules = new ArrayList(); - for (FirewallRule rule : staticNatFirewallRules) { - staticNatRules.add(_rulesMgr.buildStaticNatRule(rule, false)); - } - createApplyStaticNatRulesCommands(staticNatRules, router, cmds, guestNetworkId); - } - - // Re-apply vpn rules - s_logger.debug("Found " + vpns.size() + " vpn(s) to apply as a part of domR " + router + " start."); - if (!vpns.isEmpty()) { - for (RemoteAccessVpn vpn : vpns) { - createApplyVpnCommands(vpn, router, cmds); - } - } - - List lbs = _loadBalancerDao.listByNetworkId(guestNetworkId); - List lbRules = new ArrayList(); - if (_networkMgr.isProviderSupportServiceInNetwork(guestNetworkId, Service.Lb, provider)) { - // Re-apply load balancing rules - for (LoadBalancerVO lb : lbs) { - List dstList = _lbMgr.getExistingDestinations(lb.getId()); - List policyList = _lbMgr.getStickinessPolicies(lb.getId()); - LoadBalancingRule loadBalancing = new LoadBalancingRule(lb, dstList, policyList); - lbRules.add(loadBalancing); - } - } - - s_logger.debug("Found " + lbRules.size() + " load balancing rule(s) to apply as a part of domR " + router + " start."); - if (!lbRules.isEmpty()) { - createApplyLoadBalancingRulesCommands(lbRules, router, cmds, guestNetworkId); - } - } - } - - protected void finalizeIpAssocForNetwork(Commands cmds, VirtualRouter router, Provider provider, - Long guestNetworkId, Map vlanMacAddress) { - - ArrayList publicIps = getPublicIpsToApply(router, provider, guestNetworkId); - - if (publicIps != null && !publicIps.isEmpty()) { - s_logger.debug("Found " + publicIps.size() + " ip(s) to apply as a part of domR " + router + " start."); - // Re-apply public ip addresses - should come before PF/LB/VPN - if (_networkMgr.isProviderSupportServiceInNetwork(guestNetworkId, Service.Firewall, provider)) { - createAssociateIPCommands(router, publicIps, cmds, 0); - } - } - } - - protected ArrayList getPublicIpsToApply(VirtualRouter router, Provider provider, - Long guestNetworkId, com.cloud.network.IpAddress.State... skipInStates) { - long ownerId = router.getAccountId(); - final List userIps = _networkMgr.listPublicIpsAssignedToGuestNtwk(ownerId, guestNetworkId, null); - List allPublicIps = new ArrayList(); - if (userIps != null && !userIps.isEmpty()) { - boolean addIp = true; - for (IPAddressVO userIp : userIps) { - if (skipInStates != null) { - for (IpAddress.State stateToSkip : skipInStates) { - if (userIp.getState() == stateToSkip) { - s_logger.debug("Skipping ip address " + userIp + " in state " + userIp.getState()); - addIp = false; - break; - } - } - } - - if (addIp) { - PublicIp publicIp = new PublicIp(userIp, _vlanDao.findById(userIp.getVlanId()), - NetUtils.createSequenceBasedMacAddress(userIp.getMacAddress())); - allPublicIps.add(publicIp); - } - } - } - - //Get public Ips that should be handled by router - Network network = _networkDao.findById(guestNetworkId); - Map> ipToServices = _networkMgr.getIpToServices(allPublicIps, false, false); - Map> providerToIpList = _networkMgr.getProviderToIpList(network, ipToServices); - // Only cover virtual router for now, if ELB use it this need to be modified - - ArrayList publicIps = providerToIpList.get(provider); - return publicIps; - } - - @Override - public boolean finalizeStart(VirtualMachineProfile profile, long hostId, Commands cmds, - ReservationContext context) { - DomainRouterVO router = profile.getVirtualMachine(); - - boolean result = true; - - Answer answer = cmds.getAnswer("checkSsh"); - if (answer != null && answer instanceof CheckSshAnswer) { - CheckSshAnswer sshAnswer = (CheckSshAnswer) answer; - if (sshAnswer == null || !sshAnswer.getResult()) { - s_logger.warn("Unable to ssh to the VM: " + sshAnswer.getDetails()); - result = false; - } - } else { - result = false; - } - if (result == false) { - return result; - } - - //Get guest networks info - List guestNetworks = new ArrayList(); - - List routerNics = _nicDao.listByVmId(profile.getId()); - for (Nic routerNic : routerNics) { - Network network = _networkMgr.getNetwork(routerNic.getNetworkId()); - if (network.getTrafficType() == TrafficType.Guest) { - guestNetworks.add(network); - } - } - - answer = cmds.getAnswer("getDomRVersion"); - if (answer != null && answer instanceof GetDomRVersionAnswer) { - GetDomRVersionAnswer versionAnswer = (GetDomRVersionAnswer)answer; - if (answer == null || !answer.getResult()) { - s_logger.warn("Unable to get the template/scripts version of router " + router.getInstanceName() + - " due to: " + versionAnswer.getDetails()); - result = false; - } else { - router.setTemplateVersion(versionAnswer.getTemplateVersion()); - router.setScriptsVersion(versionAnswer.getScriptsVersion()); - router = _routerDao.persist(router, guestNetworks); - } - } else { - result = false; - } - - return result; - } - - @Override - public void finalizeStop(VirtualMachineProfile profile, StopAnswer answer) { - if (answer != null) { - VMInstanceVO vm = profile.getVirtualMachine(); - DomainRouterVO domR = _routerDao.findById(vm.getId()); - processStopOrRebootAnswer(domR, answer); - } - } - - @Override - public void finalizeExpunge(DomainRouterVO vm) { - } - - - @Override - public boolean startRemoteAccessVpn(Network network, RemoteAccessVpn vpn, List routers) - throws ResourceUnavailableException { - if (routers == null || routers.isEmpty()) { - s_logger.warn("Failed to start remote access VPN: no router found for account and zone"); - throw new ResourceUnavailableException("Failed to start remote access VPN: no router found for account and zone", - DataCenter.class, network.getDataCenterId()); - } - - for (VirtualRouter router : routers) { - if (router.getState() != State.Running) { - s_logger.warn("Failed to start remote access VPN: router not in right state " + router.getState()); - throw new ResourceUnavailableException("Failed to start remote access VPN: router not in right state " - + router.getState(), DataCenter.class, network.getDataCenterId()); - } - - Commands cmds = new Commands(OnError.Stop); - createApplyVpnCommands(vpn, router, cmds); - - try { - _agentMgr.send(router.getHostId(), cmds); - } catch (OperationTimedoutException e) { - s_logger.debug("Failed to start remote access VPN: ", e); - throw new AgentUnavailableException("Unable to send commands to virtual router ", router.getHostId(), e); - } - Answer answer = cmds.getAnswer("users"); - if (!answer.getResult()) { - s_logger.error("Unable to start vpn: unable add users to vpn in zone " + router.getDataCenterIdToDeployIn() - + " for account " + vpn.getAccountId() + " on domR: " + router.getInstanceName() - + " due to " + answer.getDetails()); - throw new ResourceUnavailableException("Unable to start vpn: Unable to add users to vpn in zone " + - router.getDataCenterIdToDeployIn() + " for account " + vpn.getAccountId() + " on domR: " - + router.getInstanceName() + " due to " + answer.getDetails(), DataCenter.class, router.getDataCenterIdToDeployIn()); - } - answer = cmds.getAnswer("startVpn"); - if (!answer.getResult()) { - s_logger.error("Unable to start vpn in zone " + router.getDataCenterIdToDeployIn() + " for account " + - vpn.getAccountId() + " on domR: " + router.getInstanceName() + " due to " - + answer.getDetails()); - throw new ResourceUnavailableException("Unable to start vpn in zone " + router.getDataCenterIdToDeployIn() - + " for account " + vpn.getAccountId() + " on domR: " + router.getInstanceName() - + " due to " + answer.getDetails(), DataCenter.class, router.getDataCenterIdToDeployIn()); - } - - } - return true; - } - - - @Override - public boolean deleteRemoteAccessVpn(Network network, RemoteAccessVpn vpn, List routers) - throws ResourceUnavailableException { - if (routers == null || routers.isEmpty()) { - s_logger.warn("Failed to delete remote access VPN: no router found for account and zone"); - throw new ResourceUnavailableException("Failed to delete remote access VPN", DataCenter.class, network.getDataCenterId()); - } - - boolean result = true; - for (VirtualRouter router : routers) { - if (router.getState() == State.Running) { - Commands cmds = new Commands(OnError.Continue); - IpAddress ip = _networkMgr.getIp(vpn.getServerAddressId()); - - RemoteAccessVpnCfgCommand removeVpnCmd = new RemoteAccessVpnCfgCommand(false, ip.getAddress().addr(), - vpn.getLocalIp(), vpn.getIpRange(), vpn.getIpsecPresharedKey()); - removeVpnCmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); - removeVpnCmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(network.getId(), router.getId())); - removeVpnCmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); - - DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); - removeVpnCmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); - - cmds.addCommand(removeVpnCmd); - - result = result && sendCommandsToRouter(router, cmds); - } else if (router.getState() == State.Stopped) { - s_logger.debug("Router " + router + " is in Stopped state, not sending deleteRemoteAccessVpn command to it"); - continue; - } else { - s_logger.warn("Failed to delete remote access VPN: domR " + router + " is not in right state " + router.getState()); - throw new ResourceUnavailableException("Failed to delete remote access VPN: domR is not in right state " + - router.getState(), DataCenter.class, network.getDataCenterId()); - } - } - - return result; - } - - - private DomainRouterVO start(DomainRouterVO router, User user, Account caller, Map params, DeploymentPlan planToDeploy) - throws StorageUnavailableException, InsufficientCapacityException, - ConcurrentOperationException, ResourceUnavailableException { - s_logger.debug("Starting router " + router); - if (_itMgr.start(router, params, user, caller, planToDeploy) != null) { - // We don't want the failure of VPN Connection affect the status of router, so we try to make connection only after router start successfully + if (conn.getState() != Site2SiteVpnConnection.State.Connected && + conn.getState() != Site2SiteVpnConnection.State.Disconnected) { + continue; + } + Site2SiteVpnConnection.State oldState = conn.getState(); + Site2SiteCustomerGateway gw = _s2sCustomerGatewayDao.findById(conn.getCustomerGatewayId()); + if (answer.isConnected(gw.getGatewayIp())) { + conn.setState(Site2SiteVpnConnection.State.Connected); + } else { + conn.setState(Site2SiteVpnConnection.State.Disconnected); + } + _s2sVpnConnectionDao.persist(conn); + if (oldState != conn.getState()) { + String title = "Site-to-site Vpn Connection to " + gw.getName() + + " just switch from " + oldState + " to " + conn.getState(); + String context = "Site-to-site Vpn Connection to " + gw.getName() + " on router " + router.getHostName() + + "(id: " + router.getId() + ") " + " just switch from " + oldState + " to " + conn.getState(); + s_logger.info(context); + _alertMgr.sendAlert(AlertManager.ALERT_TYPE_DOMAIN_ROUTER, + router.getDataCenterIdToDeployIn(), router.getPodIdToDeployIn(), title, context); + } + } finally { + _s2sVpnConnectionDao.releaseFromLockTable(lock.getId()); + } + } + } + } + } + + protected void updateRoutersRedundantState(List routers) { + boolean updated = false; + for (DomainRouterVO router : routers) { + updated = false; + if (!router.getIsRedundantRouter()) { + continue; + } + RedundantState prevState = router.getRedundantState(); + if (router.getState() != State.Running) { + router.setRedundantState(RedundantState.UNKNOWN); + router.setIsPriorityBumpUp(false); + updated = true; + } else { + String privateIP = router.getPrivateIpAddress(); + HostVO host = _hostDao.findById(router.getHostId()); + if (host == null || host.getStatus() != Status.Up) { + router.setRedundantState(RedundantState.UNKNOWN); + updated = true; + } else if (host.getManagementServerId() != ManagementServerNode.getManagementServerId()) { + /* Only cover hosts managed by this management server */ + continue; + } else if (privateIP != null) { + final CheckRouterCommand command = new CheckRouterCommand(); + command.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); + command.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); + command.setWait(60); + final Answer origAnswer = _agentMgr.easySend(router.getHostId(), command); + CheckRouterAnswer answer = null; + if (origAnswer instanceof CheckRouterAnswer) { + answer = (CheckRouterAnswer) origAnswer; + } else { + s_logger.warn("Unable to update router " + router.getHostName() + "'s status"); + } + RedundantState state = RedundantState.UNKNOWN; + boolean isBumped = router.getIsPriorityBumpUp(); + if (answer != null && answer.getResult()) { + state = answer.getState(); + isBumped = answer.isBumped(); + } + router.setRedundantState(state); + router.setIsPriorityBumpUp(isBumped); + updated = true; + } + } + if (updated) { + Transaction txn = Transaction.open(Transaction.CLOUD_DB); + try { + txn.start(); + _routerDao.update(router.getId(), router); + txn.commit(); + } catch (Exception e) { + txn.rollback(); + s_logger.warn("Unable to update router status for account: " + router.getAccountId()); + } finally { + txn.close(); + } + } + RedundantState currState = router.getRedundantState(); + if (prevState != currState) { + String title = "Redundant virtual router " + router.getInstanceName() + + " just switch from " + prevState + " to " + currState; + String context = "Redundant virtual router (name: " + router.getHostName() + ", id: " + router.getId() + ") " + + " just switch from " + prevState + " to " + currState; + s_logger.info(context); + if (currState == RedundantState.MASTER) { + _alertMgr.sendAlert(AlertManager.ALERT_TYPE_DOMAIN_ROUTER, + router.getDataCenterIdToDeployIn(), router.getPodIdToDeployIn(), title, context); + } + } + } + } + + // Ensure router status is update to date before execute this function. The function would try best to recover all +// routers except MASTER + protected void recoverRedundantNetwork(DomainRouterVO masterRouter, DomainRouterVO backupRouter) { + UserContext context = UserContext.current(); + context.setAccountId(1); + if (masterRouter.getState() == State.Running && backupRouter.getState() == State.Running) { + HostVO masterHost = _hostDao.findById(masterRouter.getHostId()); + HostVO backupHost = _hostDao.findById(backupRouter.getHostId()); + if (masterHost.getStatus() == Status.Up && backupHost.getStatus() == Status.Up) { + String title = "Reboot " + backupRouter.getInstanceName() + " to ensure redundant virtual routers work"; + if (s_logger.isDebugEnabled()) { + s_logger.debug(title); + } + _alertMgr.sendAlert(AlertManager.ALERT_TYPE_DOMAIN_ROUTER, + backupRouter.getDataCenterIdToDeployIn(), backupRouter.getPodIdToDeployIn(), title, title); + try { + rebootRouter(backupRouter.getId(), false); + } catch (ConcurrentOperationException e) { + s_logger.warn("Fail to reboot " + backupRouter.getInstanceName(), e); + } catch (ResourceUnavailableException e) { + s_logger.warn("Fail to reboot " + backupRouter.getInstanceName(), e); + } catch (InsufficientCapacityException e) { + s_logger.warn("Fail to reboot " + backupRouter.getInstanceName(), e); + } + } + } + } + + private int getRealPriority(DomainRouterVO router) { + int priority = router.getPriority(); + if (router.getIsPriorityBumpUp()) { + priority += DEFAULT_DELTA; + } + return priority; + } + + protected class CheckRouterTask implements Runnable { + + public CheckRouterTask() { + } + + /* + * In order to make fail-over works well at any time, we have to ensure: + * 1. Backup router's priority = Master's priority - DELTA + 1 + * 2. Backup router's priority hasn't been bumped up. + */ + private void checkSanity(List routers) { + Set checkedNetwork = new HashSet(); + for (DomainRouterVO router : routers) { + if (!router.getIsRedundantRouter()) { + continue; + } + + List routerGuestNtwkIds = _routerDao.getRouterNetworks(router.getId()); + + for (Long routerGuestNtwkId : routerGuestNtwkIds) { + if (checkedNetwork.contains(routerGuestNtwkId)) { + continue; + } + checkedNetwork.add(routerGuestNtwkId); + List checkingRouters = _routerDao.listByNetworkAndRole(routerGuestNtwkId, Role.VIRTUAL_ROUTER); + if (checkingRouters.size() != 2) { + continue; + } + DomainRouterVO masterRouter = null; + DomainRouterVO backupRouter = null; + for (DomainRouterVO r : checkingRouters) { + if (r.getRedundantState() == RedundantState.MASTER) { + if (masterRouter == null) { + masterRouter = r; + } else { + // Duplicate master! We give up, until the admin fix duplicate MASTER issue + break; + } + } else if (r.getRedundantState() == RedundantState.BACKUP) { + if (backupRouter == null) { + backupRouter = r; + } else { + break; + } + } + } + if (masterRouter != null && backupRouter != null) { + if (getRealPriority(masterRouter) - DEFAULT_DELTA + 1 != getRealPriority(backupRouter) || backupRouter.getIsPriorityBumpUp()) { + recoverRedundantNetwork(masterRouter, backupRouter); + } + } + } + } + } + + private void checkDuplicateMaster(List routers) { + Map networkRouterMaps = new HashMap(); + for (DomainRouterVO router : routers) { + List routerGuestNtwkIds = _routerDao.getRouterNetworks(router.getId()); + + for (Long routerGuestNtwkId : routerGuestNtwkIds) { + if (router.getRedundantState() == RedundantState.MASTER) { + if (networkRouterMaps.containsKey(routerGuestNtwkId)) { + DomainRouterVO dupRouter = networkRouterMaps.get(routerGuestNtwkId); + String title = "More than one redundant virtual router is in MASTER state! Router " + router.getHostName() + " and router " + dupRouter.getHostName(); + String context = "Virtual router (name: " + router.getHostName() + ", id: " + router.getId() + " and router (name: " + + dupRouter.getHostName() + ", id: " + router.getId() + ") are both in MASTER state! If the problem persist, restart both of routers. "; + + _alertMgr.sendAlert(AlertManager.ALERT_TYPE_DOMAIN_ROUTER, router.getDataCenterIdToDeployIn(), router.getPodIdToDeployIn(), title, context); + _alertMgr.sendAlert(AlertManager.ALERT_TYPE_DOMAIN_ROUTER, dupRouter.getDataCenterIdToDeployIn(), dupRouter.getPodIdToDeployIn(), title, context); + } else { + networkRouterMaps.put(routerGuestNtwkId, router); + } + } + } + } + } + + @Override + public void run() { + try { + final List routers = _routerDao.listIsolatedByHostId(null); + s_logger.debug("Found " + routers.size() + " routers. "); + + updateRoutersRedundantState(routers); + updateSite2SiteVpnConnectionState(routers); + + /* + * FIXME assumed the a pair of redundant routers managed by same mgmt server, + * then the update above can get the latest status + */ + checkDuplicateMaster(routers); + checkSanity(routers); + } catch (Exception ex) { + s_logger.error("Fail to complete the CheckRouterTask! ", ex); + } + } + } + + private final int DEFAULT_PRIORITY = 100; + private final int DEFAULT_DELTA = 2; + + protected int getUpdatedPriority(Network guestNetwork, List routers, DomainRouterVO exclude) throws InsufficientVirtualNetworkCapcityException { + int priority; + if (routers.size() == 0) { + priority = DEFAULT_PRIORITY; + } else { + int maxPriority = 0; + for (DomainRouterVO r : routers) { + if (!r.getIsRedundantRouter()) { + throw new CloudRuntimeException("Redundant router is mixed with single router in one network!"); + } + // FIXME Assume the maxPriority one should be running or just created. + if (r.getId() != exclude.getId() && getRealPriority(r) > maxPriority) { + maxPriority = getRealPriority(r); + } + } + if (maxPriority == 0) { + return DEFAULT_PRIORITY; + } + if (maxPriority < 20) { + s_logger.error("Current maximum priority is too low!"); + throw new InsufficientVirtualNetworkCapcityException("Current maximum priority is too low as " + maxPriority + "!", + guestNetwork.getId()); + } else if (maxPriority > 200) { + s_logger.error("Too many times fail-over happened! Current maximum priority is too high as " + maxPriority + "!"); + throw new InsufficientVirtualNetworkCapcityException("Too many times fail-over happened! Current maximum priority is too high as " + + maxPriority + "!", guestNetwork.getId()); + } + priority = maxPriority - DEFAULT_DELTA + 1; + } + return priority; + } + + /* + * Ovm won't support any system. So we have to choose a partner cluster in the same pod to start domain router for + * us + */ + private HypervisorType getClusterToStartDomainRouterForOvm(long podId) { + List clusters = _clusterDao.listByPodId(podId); + for (ClusterVO cv : clusters) { + if (cv.getHypervisorType() == HypervisorType.Ovm || cv.getHypervisorType() == HypervisorType.BareMetal) { + continue; + } + + List hosts = _resourceMgr.listAllHostsInCluster(cv.getId()); + if (hosts == null || hosts.isEmpty()) { + continue; + } + + for (HostVO h : hosts) { + if (h.getStatus() == Status.Up) { + s_logger.debug("Pick up host that has hypervisor type " + h.getHypervisorType() + " in cluster " + + cv.getId() + " to start domain router for OVM"); + return h.getHypervisorType(); + } + } + } + + String errMsg = "Cannot find an available cluster in Pod " + + podId + + " to start domain router for Ovm. \n Ovm won't support any system vm including domain router, " + + "please make sure you have a cluster with hypervisor type of any of xenserver/KVM/Vmware in the same pod" + + " with Ovm cluster. And there is at least one host in UP status in that cluster."; + throw new CloudRuntimeException(errMsg); + } + + @DB + protected List findOrDeployVirtualRouterInGuestNetwork(Network guestNetwork, DeployDestination dest, Account owner, + boolean isRedundant, Map params) throws ConcurrentOperationException, + InsufficientCapacityException, ResourceUnavailableException { + + assert guestNetwork.getState() == Network.State.Implemented || guestNetwork.getState() == Network.State.Setup || + guestNetwork.getState() == Network.State.Implementing : "Network is not yet fully implemented: " + + guestNetwork; + assert guestNetwork.getTrafficType() == TrafficType.Guest; + + // 1) Get deployment plan and find out the list of routers + boolean isPodBased = (dest.getDataCenter().getNetworkType() == NetworkType.Basic || + _networkMgr.areServicesSupportedInNetwork(guestNetwork.getId(), Service.SecurityGroup)) + && guestNetwork.getTrafficType() == TrafficType.Guest; + + Pair> planAndRouters = getDeploymentPlanAndRouters(isPodBased, dest, guestNetwork.getId()); + DeploymentPlan plan = planAndRouters.first(); + List routers = planAndRouters.second(); + + // 2) Figure out required routers count + int routerCount = 1; + if (isRedundant) { + routerCount = 2; + } + + /* If it is the single router network, then keep it untouched */ + for (DomainRouterVO router : routers) { + if (!router.getIsRedundantRouter() || isPodBased) { + routerCount = 1; + break; + } + } + + /* If old network is redundant but new is single router, then routers.size() = 2 but routerCount = 1 */ + if (routers.size() >= routerCount) { + return routers; + } + + if (routers.size() >= 5) { + s_logger.error("Too much redundant routers!"); + } + + Network network = _networkDao.acquireInLockTable(guestNetwork.getId()); + if (network == null) { + throw new ConcurrentOperationException("Unable to lock network " + guestNetwork.getId()); + } + + try { + // Check if providers are supported in the physical networks + VirtualRouterProviderType type = VirtualRouterProviderType.VirtualRouter; + Long physicalNetworkId = _networkMgr.getPhysicalNetworkId(network); + PhysicalNetworkServiceProvider provider = _physicalProviderDao.findByServiceProvider(physicalNetworkId, type.toString()); + if (provider == null) { + throw new CloudRuntimeException("Cannot find service provider " + type.toString() + " in physical network " + physicalNetworkId); + } + VirtualRouterProvider vrProvider = _vrProviderDao.findByNspIdAndType(provider.getId(), type); + if (vrProvider == null) { + throw new CloudRuntimeException("Cannot find virtual router provider " + type.toString() + " as service provider " + provider.getId()); + } + + if (_networkMgr.isNetworkSystem(guestNetwork) || guestNetwork.getGuestType() == Network.GuestType.Shared) { + owner = _accountMgr.getAccount(Account.ACCOUNT_ID_SYSTEM); + } + + // Check if public network has to be set on VR + boolean publicNetwork = false; + if (_networkMgr.isProviderSupportServiceInNetwork(guestNetwork.getId(), Service.SourceNat, Provider.VirtualRouter)) { + publicNetwork = true; + } + if (isRedundant && !publicNetwork) { + s_logger.error("Didn't support redundant virtual router without public network!"); + return null; + } + + Long offeringId = _networkOfferingDao.findById(guestNetwork.getNetworkOfferingId()).getServiceOfferingId(); + if (offeringId == null) { + offeringId = _offering.getId(); + } + + PublicIp sourceNatIp = null; + if (publicNetwork) { + sourceNatIp = _networkMgr.assignSourceNatIpAddressToGuestNetwork(owner, guestNetwork); + } + + // 3) deploy virtual router(s) + int count = routerCount - routers.size(); + for (int i = 0; i < count; i++) { + List> networks = createRouterNetworks(owner, isRedundant, plan, guestNetwork, + new Pair(publicNetwork, sourceNatIp)); + DomainRouterVO router = deployRouter(owner, dest, plan, params, isRedundant, vrProvider, offeringId, + null, networks); + + _routerDao.addRouterToGuestNetwork(router, network); + + routers.add(router); + } + } finally { + if (network != null) { + _networkDao.releaseFromLockTable(network.getId()); + } + } + return routers; + } + + protected DomainRouterVO deployRouter(Account owner, DeployDestination dest, DeploymentPlan plan, Map params, + boolean isRedundant, VirtualRouterProvider vrProvider, long svcOffId, + Long vpcId, List> networks) throws ConcurrentOperationException, + InsufficientAddressCapacityException, InsufficientServerCapacityException, InsufficientCapacityException, + StorageUnavailableException, ResourceUnavailableException { + + long id = _routerDao.getNextInSequence(Long.class, "id"); + if (s_logger.isDebugEnabled()) { + s_logger.debug("Creating the router " + id + " in datacenter " + dest.getDataCenter()); + } + + ServiceOfferingVO routerOffering = _serviceOfferingDao.findById(svcOffId); + + // Router is the network element, we don't know the hypervisor type yet. + // Try to allocate the domR twice using diff hypervisors, and when failed both times, throw the exception up + List supportedHypervisors = new ArrayList(); + HypervisorType defaults = _resourceMgr.getDefaultHypervisor(dest.getDataCenter().getId()); + if (defaults != HypervisorType.None) { + supportedHypervisors.add(defaults); + } + + if (dest.getCluster() != null) { + if (dest.getCluster().getHypervisorType() == HypervisorType.Ovm) { + supportedHypervisors.add(getClusterToStartDomainRouterForOvm(dest.getCluster().getPodId())); + } else { + supportedHypervisors.add(dest.getCluster().getHypervisorType()); + } + } else { + supportedHypervisors = _resourceMgr.getSupportedHypervisorTypes(dest.getDataCenter().getId(), true, + plan.getPodId()); + } + + if (supportedHypervisors.isEmpty()) { + if (plan.getPodId() != null) { + throw new InsufficientServerCapacityException("Unable to create virtual router, " + + "there are no clusters in the pod ", Pod.class, plan.getPodId()); + } + throw new InsufficientServerCapacityException("Unable to create virtual router, " + + "there are no clusters in the zone ", DataCenter.class, dest.getDataCenter().getId()); + } + + int allocateRetry = 0; + int startRetry = 0; + DomainRouterVO router = null; + for (Iterator iter = supportedHypervisors.iterator(); iter.hasNext();) { + HypervisorType hType = iter.next(); + try { + s_logger.debug("Allocating the domR with the hypervisor type " + hType); + VMTemplateVO template = _templateDao.findRoutingTemplate(hType); + + if (template == null) { + s_logger.debug(hType + " won't support system vm, skip it"); + continue; + } + + boolean offerHA = routerOffering.getOfferHA(); + /* + * We don't provide HA to redundant router VMs, admin should own it all, and redundant router themselves + * are HA + */ + if (isRedundant) { + offerHA = false; + } + + router = new DomainRouterVO(id, routerOffering.getId(), vrProvider.getId(), + VirtualMachineName.getRouterName(id, _instance), template.getId(), template.getHypervisorType(), + template.getGuestOSId(), owner.getDomainId(), owner.getId(), isRedundant, 0, false, + RedundantState.UNKNOWN, offerHA, false, vpcId); + router.setRole(Role.VIRTUAL_ROUTER); + router = _itMgr.allocate(router, template, routerOffering, networks, plan, null, owner); + } catch (InsufficientCapacityException ex) { + if (allocateRetry < 2 && iter.hasNext()) { + s_logger.debug("Failed to allocate the domR with hypervisor type " + hType + ", retrying one more time"); + continue; + } else { + throw ex; + } + } finally { + allocateRetry++; + } + + try { + router = startVirtualRouter(router, _accountMgr.getSystemUser(), _accountMgr.getSystemAccount(), params); + break; + } catch (InsufficientCapacityException ex) { + if (startRetry < 2 && iter.hasNext()) { + s_logger.debug("Failed to start the domR " + router + " with hypervisor type " + hType + ", " + + "destroying it and recreating one more time"); + // destroy the router + destroyRouter(router.getId()); + continue; + } else { + throw ex; + } + } finally { + startRetry++; + } + } + + return router; + } + + protected List> createRouterNetworks(Account owner, boolean isRedundant, + DeploymentPlan plan, Network guestNetwork, Pair publicNetwork) throws ConcurrentOperationException, + InsufficientAddressCapacityException { + + boolean setupPublicNetwork = false; + if (publicNetwork != null) { + setupPublicNetwork = publicNetwork.first(); + } + + // Form networks + List> networks = new ArrayList>(3); + + // 1) Guest network + boolean hasGuestNetwork = false; + if (guestNetwork != null) { + s_logger.debug("Adding nic for Virtual Router in Guest network " + guestNetwork); + String defaultNetworkStartIp = null; + if (guestNetwork.getCidr() != null && !setupPublicNetwork) { + String startIp = _networkMgr.getStartIpAddress(guestNetwork.getId()); + if (startIp != null && _ipAddressDao.findByIpAndSourceNetworkId(guestNetwork.getId(), startIp).getAllocatedTime() == null) { + defaultNetworkStartIp = startIp; + } else if (s_logger.isDebugEnabled()) { + s_logger.debug("First ip " + startIp + " in network id=" + guestNetwork.getId() + + " is already allocated, can't use it for domain router; will get random ip address from the range"); + } + } + + NicProfile gatewayNic = new NicProfile(defaultNetworkStartIp); + if (setupPublicNetwork) { + if (isRedundant) { + gatewayNic.setIp4Address(_networkMgr.acquireGuestIpAddress(guestNetwork, null)); + } else { + gatewayNic.setIp4Address(guestNetwork.getGateway()); + } + gatewayNic.setBroadcastUri(guestNetwork.getBroadcastUri()); + gatewayNic.setBroadcastType(guestNetwork.getBroadcastDomainType()); + gatewayNic.setIsolationUri(guestNetwork.getBroadcastUri()); + gatewayNic.setMode(guestNetwork.getMode()); + String gatewayCidr = guestNetwork.getCidr(); + gatewayNic.setNetmask(NetUtils.getCidrNetmask(gatewayCidr)); + } else { + gatewayNic.setDefaultNic(true); + } + + networks.add(new Pair((NetworkVO) guestNetwork, gatewayNic)); + hasGuestNetwork = true; + } + + // 2) Control network + s_logger.debug("Adding nic for Virtual Router in Control network "); + List offerings = _networkMgr.getSystemAccountNetworkOfferings(NetworkOfferingVO.SystemControlNetwork); + NetworkOfferingVO controlOffering = offerings.get(0); + NetworkVO controlConfig = _networkMgr.setupNetwork(_systemAcct, controlOffering, plan, null, null, false).get(0); + networks.add(new Pair(controlConfig, null)); + + // 3) Public network + if (setupPublicNetwork) { + PublicIp sourceNatIp = publicNetwork.second(); + s_logger.debug("Adding nic for Virtual Router in Public network "); + // if source nat service is supported by the network, get the source nat ip address + NicProfile defaultNic = new NicProfile(); + defaultNic.setDefaultNic(true); + defaultNic.setIp4Address(sourceNatIp.getAddress().addr()); + defaultNic.setGateway(sourceNatIp.getGateway()); + defaultNic.setNetmask(sourceNatIp.getNetmask()); + defaultNic.setMacAddress(sourceNatIp.getMacAddress()); + defaultNic.setBroadcastType(BroadcastDomainType.Vlan); + defaultNic.setBroadcastUri(BroadcastDomainType.Vlan.toUri(sourceNatIp.getVlanTag())); + defaultNic.setIsolationUri(IsolationType.Vlan.toUri(sourceNatIp.getVlanTag())); + if (hasGuestNetwork) { + defaultNic.setDeviceId(2); + } + NetworkOfferingVO publicOffering = _networkMgr.getSystemAccountNetworkOfferings(NetworkOfferingVO.SystemPublicNetwork).get(0); + List publicNetworks = _networkMgr.setupNetwork(_systemAcct, publicOffering, plan, null, null, false); + networks.add(new Pair(publicNetworks.get(0), defaultNic)); + } + + return networks; + } + + protected Pair> getDeploymentPlanAndRouters(boolean isPodBased, + DeployDestination dest, long guestNetworkId) { + long dcId = dest.getDataCenter().getId(); + List routers = null; + DeploymentPlan plan = new DataCenterDeployment(dcId); + if (isPodBased) { + Pod pod = dest.getPod(); + Long podId = null; + if (pod != null) { + podId = pod.getId(); + } else { + throw new CloudRuntimeException("Pod id is expected in deployment destination"); + } + routers = _routerDao.listByNetworkAndPodAndRole(guestNetworkId, podId, Role.VIRTUAL_ROUTER); + plan = new DataCenterDeployment(dcId, podId, null, null, null, null); + } else { + routers = _routerDao.listByNetworkAndRole(guestNetworkId, Role.VIRTUAL_ROUTER); + } + + return new Pair>(plan, routers); + } + + private DomainRouterVO startVirtualRouter(DomainRouterVO router, User user, Account caller, Map params) + throws StorageUnavailableException, InsufficientCapacityException, + ConcurrentOperationException, ResourceUnavailableException { + + if (router.getRole() != Role.VIRTUAL_ROUTER || !router.getIsRedundantRouter()) { + return this.start(router, user, caller, params, null); + } + + if (router.getState() == State.Running) { + s_logger.debug("Redundant router " + router.getInstanceName() + " is already running!"); + return router; + } + + DataCenterDeployment plan = new DataCenterDeployment(0, null, null, null, null, null); + DomainRouterVO result = null; + assert router.getIsRedundantRouter(); + List routerList = _routerDao.findBy(router.getAccountId(), router.getDataCenterIdToDeployIn()); + DomainRouterVO routerToBeAvoid = null; + for (DomainRouterVO rrouter : routerList) { + if (rrouter.getHostId() != null && rrouter.getIsRedundantRouter() && rrouter.getState() == State.Running) { + if (routerToBeAvoid != null) { + throw new ResourceUnavailableException("Try to start router " + router.getInstanceName() + "(" + router.getId() + ")" + + ", but there are already two redundant routers with IP " + router.getPublicIpAddress() + + ", they are " + rrouter.getInstanceName() + "(" + rrouter.getId() + ") and " + + routerToBeAvoid.getInstanceName() + "(" + routerToBeAvoid.getId() + ")", + DataCenter.class, rrouter.getDataCenterIdToDeployIn()); + } + routerToBeAvoid = rrouter; + } + } + if (routerToBeAvoid == null) { + return this.start(router, user, caller, params, null); + } + // We would try best to deploy the router to another place + int retryIndex = 5; + ExcludeList[] avoids = new ExcludeList[5]; + avoids[0] = new ExcludeList(); + avoids[0].addPod(routerToBeAvoid.getPodIdToDeployIn()); + avoids[1] = new ExcludeList(); + avoids[1].addCluster(_hostDao.findById(routerToBeAvoid.getHostId()).getClusterId()); + avoids[2] = new ExcludeList(); + List volumes = _volumeDao.findByInstanceAndType(routerToBeAvoid.getId(), Type.ROOT); + if (volumes != null && volumes.size() != 0) { + avoids[2].addPool(volumes.get(0).getPoolId()); + } + avoids[2].addHost(routerToBeAvoid.getHostId()); + avoids[3] = new ExcludeList(); + avoids[3].addHost(routerToBeAvoid.getHostId()); + avoids[4] = new ExcludeList(); + + for (int i = 0; i < retryIndex; i++) { + if (s_logger.isTraceEnabled()) { + s_logger.trace("Try to deploy redundant virtual router:" + router.getHostName() + ", for " + i + " time"); + } + plan.setAvoids(avoids[i]); + try { + result = this.start(router, user, caller, params, plan); + } catch (InsufficientServerCapacityException ex) { + result = null; + } + if (result != null) { + break; + } + } + return result; + } + + @Override + public List deployVirtualRouterInGuestNetwork(Network guestNetwork, DeployDestination dest, Account owner, + Map params, boolean isRedundant) throws InsufficientCapacityException, + ConcurrentOperationException, ResourceUnavailableException { + + List routers = findOrDeployVirtualRouterInGuestNetwork + (guestNetwork, dest, owner, isRedundant, params); + + return startRouters(params, routers); + } + + protected List startRouters(Map params, List routers) throws StorageUnavailableException, InsufficientCapacityException, ConcurrentOperationException, + ResourceUnavailableException { + List runningRouters = null; + + if (routers != null) { + runningRouters = new ArrayList(); + } + + for (DomainRouterVO router : routers) { + boolean skip = false; + State state = router.getState(); + if (router.getHostId() != null && state != State.Running) { + HostVO host = _hostDao.findById(router.getHostId()); + if (host == null || host.getStatus() != Status.Up) { + skip = true; + } + } + if (!skip) { + if (state != State.Running) { + router = startVirtualRouter(router, _accountMgr.getSystemUser(), _accountMgr.getSystemAccount(), params); + } + if (router != null) { + runningRouters.add(router); + } + } + } + return runningRouters; + } + + @Override + public boolean finalizeVirtualMachineProfile(VirtualMachineProfile profile, DeployDestination dest, + ReservationContext context) { + + boolean dnsProvided = true; + boolean dhcpProvided = true; + boolean publicNetwork = false; + DataCenterVO dc = _dcDao.findById(dest.getDataCenter().getId()); + _dcDao.loadDetails(dc); + + // 1) Set router details + DomainRouterVO router = profile.getVirtualMachine(); + Map details = _vmDetailsDao.findDetails(router.getId()); + router.setDetails(details); + + // 2) Prepare boot loader elements related with Control network + + StringBuilder buf = profile.getBootArgsBuilder(); + buf.append(" template=domP"); + buf.append(" name=").append(profile.getHostName()); + + if (Boolean.valueOf(_configDao.getValue("system.vm.random.password"))) { + buf.append(" vmpassword=").append(_configDao.getValue("system.vm.password")); + } + + NicProfile controlNic = null; + String defaultDns1 = null; + String defaultDns2 = null; + for (NicProfile nic : profile.getNics()) { + int deviceId = nic.getDeviceId(); + buf.append(" eth").append(deviceId).append("ip=").append(nic.getIp4Address()); + buf.append(" eth").append(deviceId).append("mask=").append(nic.getNetmask()); + + if (nic.isDefaultNic()) { + buf.append(" gateway=").append(nic.getGateway()); + defaultDns1 = nic.getDns1(); + defaultDns2 = nic.getDns2(); + } + + if (nic.getTrafficType() == TrafficType.Management) { + buf.append(" localgw=").append(dest.getPod().getGateway()); + } else if (nic.getTrafficType() == TrafficType.Control) { + controlNic = nic; + // DOMR control command is sent over management server in VMware + if (dest.getHost().getHypervisorType() == HypervisorType.VMware) { + if (s_logger.isInfoEnabled()) { + s_logger.info("Check if we need to add management server explicit route to DomR. pod cidr: " + + dest.getPod().getCidrAddress() + "/" + dest.getPod().getCidrSize() + + ", pod gateway: " + dest.getPod().getGateway() + ", management host: " + _mgmt_host); + } + + if (s_logger.isInfoEnabled()) { + s_logger.info("Add management server explicit route to DomR."); + } + + // always add management explicit route, for basic networking setup, DomR may have two interfaces +// while both + // are on the same subnet + _mgmt_cidr = _configDao.getValue(Config.ManagementNetwork.key()); + if (NetUtils.isValidCIDR(_mgmt_cidr)) { + buf.append(" mgmtcidr=").append(_mgmt_cidr); + buf.append(" localgw=").append(dest.getPod().getGateway()); + } + + if (dc.getNetworkType() == NetworkType.Basic) { + // ask domR to setup SSH on guest network + buf.append(" sshonguest=true"); + } + + } + } else if (nic.getTrafficType() == TrafficType.Guest) { + dnsProvided = _networkMgr.isProviderSupportServiceInNetwork(nic.getNetworkId(), Service.Dns, Provider.VirtualRouter); + dhcpProvided = _networkMgr.isProviderSupportServiceInNetwork(nic.getNetworkId(), Service.Dhcp, Provider.VirtualRouter); + // build bootloader parameter for the guest + buf.append(createGuestBootLoadArgs(nic, defaultDns1, defaultDns2, router)); + } else if (nic.getTrafficType() == TrafficType.Public) { + publicNetwork = true; + } + } + + if (controlNic == null) { + throw new CloudRuntimeException("Didn't start a control port"); + } + + String rpValue = _configDao.getValue(Config.NetworkRouterRpFilter.key()); + if (rpValue != null && rpValue.equalsIgnoreCase("true")) { + _disable_rp_filter = true; + } else { + _disable_rp_filter = false; + } + + String rpFilter = " "; + String type = null; + if (router.getVpcId() != null) { + type = "vpcrouter"; + if (_disable_rp_filter) { + rpFilter = " disable_rp_filter=true"; + } + } else if (!publicNetwork) { + type = "dhcpsrvr"; + } else { + type = "router"; + if (_disable_rp_filter) { + rpFilter = " disable_rp_filter=true"; + } + } + + if (_disable_rp_filter) { + rpFilter = " disable_rp_filter=true"; + } + + buf.append(" type=" + type + rpFilter); + + String domain_suffix = dc.getDetail(ZoneConfig.DnsSearchOrder.getName()); + if (domain_suffix != null) { + buf.append(" dnssearchorder=").append(domain_suffix); + } + + if (profile.getHypervisorType() == HypervisorType.VMware) { + buf.append(" extra_pubnics=" + _routerExtraPublicNics); + } + + /* + * If virtual router didn't provide DNS service but provide DHCP service, we need to override the DHCP response + * to return DNS server rather than + * virtual router itself. + */ + if (dnsProvided || dhcpProvided) { + if (defaultDns1 != null) { + buf.append(" dns1=").append(defaultDns1); + } + if (defaultDns2 != null) { + buf.append(" dns2=").append(defaultDns2); + } + + boolean useExtDns = !dnsProvided; + /* For backward compatibility */ + String use_external_dns = _configDao.getValue(Config.UseExternalDnsServers.key()); + if (use_external_dns != null && use_external_dns.equals("true")) { + useExtDns = true; + } + + if (useExtDns) { + buf.append(" useextdns=true"); + } + } + + if (s_logger.isDebugEnabled()) { + s_logger.debug("Boot Args for " + profile + ": " + buf.toString()); + } + + return true; + } + + protected StringBuilder createGuestBootLoadArgs(NicProfile guestNic, String defaultDns1, + String defaultDns2, DomainRouterVO router) { + long guestNetworkId = guestNic.getNetworkId(); + NetworkVO guestNetwork = _networkDao.findById(guestNetworkId); + String dhcpRange = null; + DataCenterVO dc = _dcDao.findById(guestNetwork.getDataCenterId()); + + StringBuilder buf = new StringBuilder(); + + boolean isRedundant = router.getIsRedundantRouter(); + if (isRedundant) { + buf.append(" redundant_router=1"); + List routers = _routerDao.listByNetworkAndRole(guestNetwork.getId(), Role.VIRTUAL_ROUTER); + try { + int priority = getUpdatedPriority(guestNetwork, routers, router); + router.setPriority(priority); + } catch (InsufficientVirtualNetworkCapcityException e) { + s_logger.error("Failed to get update priority!", e); + throw new CloudRuntimeException("Failed to get update priority!"); + } + Network net = _networkMgr.getNetwork(guestNic.getNetworkId()); + buf.append(" guestgw=").append(net.getGateway()); + String brd = NetUtils.long2Ip(NetUtils.ip2Long(guestNic.getIp4Address()) | ~NetUtils.ip2Long(guestNic.getNetmask())); + buf.append(" guestbrd=").append(brd); + buf.append(" guestcidrsize=").append(NetUtils.getCidrSize(guestNic.getNetmask())); + buf.append(" router_pr=").append(router.getPriority()); + } + + // setup network domain + String domain = guestNetwork.getNetworkDomain(); + if (domain != null) { + buf.append(" domain=" + domain); + } + + // setup dhcp range + if (dc.getNetworkType() == NetworkType.Basic) { + if (guestNic.isDefaultNic()) { + long cidrSize = NetUtils.getCidrSize(guestNic.getNetmask()); + String cidr = NetUtils.getCidrSubNet(guestNic.getGateway(), cidrSize); + if (cidr != null) { + dhcpRange = NetUtils.getIpRangeStartIpFromCidr(cidr, cidrSize); + } + } + } else if (dc.getNetworkType() == NetworkType.Advanced) { + String cidr = guestNetwork.getCidr(); + if (cidr != null) { + dhcpRange = NetUtils.getDhcpRange(cidr); + } + } + + if (dhcpRange != null) { + buf.append(" dhcprange=" + dhcpRange); + } + + return buf; + } + + protected String getGuestDhcpRange(NicProfile guestNic, Network guestNetwork, DataCenter dc) { + String dhcpRange = null; + // setup dhcp range + if (dc.getNetworkType() == NetworkType.Basic) { + long cidrSize = NetUtils.getCidrSize(guestNic.getNetmask()); + String cidr = NetUtils.getCidrSubNet(guestNic.getGateway(), cidrSize); + if (cidr != null) { + dhcpRange = NetUtils.getIpRangeStartIpFromCidr(cidr, cidrSize); + } + } else if (dc.getNetworkType() == NetworkType.Advanced) { + String cidr = guestNetwork.getCidr(); + if (cidr != null) { + dhcpRange = NetUtils.getDhcpRange(cidr); + } + } + return dhcpRange; + } + + @Override + public boolean finalizeDeployment(Commands cmds, VirtualMachineProfile profile, + DeployDestination dest, ReservationContext context) throws ResourceUnavailableException { + DomainRouterVO router = profile.getVirtualMachine(); + + List nics = profile.getNics(); + for (NicProfile nic : nics) { + if (nic.getTrafficType() == TrafficType.Public) { + router.setPublicIpAddress(nic.getIp4Address()); + router.setPublicNetmask(nic.getNetmask()); + router.setPublicMacAddress(nic.getMacAddress()); + } else if (nic.getTrafficType() == TrafficType.Control) { + router.setPrivateIpAddress(nic.getIp4Address()); + router.setPrivateMacAddress(nic.getMacAddress()); + } + } + _routerDao.update(router.getId(), router); + + finalizeCommandsOnStart(cmds, profile); + return true; + } + + @Override + public boolean finalizeCommandsOnStart(Commands cmds, VirtualMachineProfile profile) { + DomainRouterVO router = profile.getVirtualMachine(); + NicProfile controlNic = getControlNic(profile); + + if (controlNic == null) { + s_logger.error("Control network doesn't exist for the router " + router); + return false; + } + + finalizeSshAndVersionAndNetworkUsageOnStart(cmds, profile, router, controlNic); + + // restart network if restartNetwork = false is not specified in profile parameters + boolean reprogramGuestNtwks = true; + if (profile.getParameter(Param.ReProgramGuestNetworks) != null + && (Boolean) profile.getParameter(Param.ReProgramGuestNetworks) == false) { + reprogramGuestNtwks = false; + } + + VirtualRouterProvider vrProvider = _vrProviderDao.findById(router.getElementId()); + if (vrProvider == null) { + throw new CloudRuntimeException("Cannot find related virtual router provider of router: " + router.getHostName()); + } + Provider provider = Network.Provider.getProvider(vrProvider.getType().toString()); + if (provider == null) { + throw new CloudRuntimeException("Cannot find related provider of virtual router provider: " + vrProvider.getType().toString()); + } + + List routerGuestNtwkIds = _routerDao.getRouterNetworks(router.getId()); + for (Long guestNetworkId : routerGuestNtwkIds) { + if (reprogramGuestNtwks) { + finalizeIpAssocForNetwork(cmds, router, provider, guestNetworkId, null); + finalizeNetworkRulesForNetwork(cmds, router, provider, guestNetworkId); + } + + finalizeUserDataAndDhcpOnStart(cmds, router, provider, guestNetworkId); + } + + return true; + } + + protected NicProfile getControlNic(VirtualMachineProfile profile) { + DomainRouterVO router = profile.getVirtualMachine(); + DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); + NicProfile controlNic = null; + if (profile.getHypervisorType() == HypervisorType.VMware && dcVo.getNetworkType() == NetworkType.Basic) { + // TODO this is a ugly to test hypervisor type here + // for basic network mode, we will use the guest NIC for control NIC + for (NicProfile nic : profile.getNics()) { + if (nic.getTrafficType() == TrafficType.Guest && nic.getIp4Address() != null) { + controlNic = nic; + } + } + } else { + for (NicProfile nic : profile.getNics()) { + if (nic.getTrafficType() == TrafficType.Control && nic.getIp4Address() != null) { + controlNic = nic; + } + } + } + return controlNic; + } + + protected void finalizeSshAndVersionAndNetworkUsageOnStart(Commands cmds, VirtualMachineProfile profile, DomainRouterVO router, NicProfile controlNic) { + cmds.addCommand("checkSsh", new CheckSshCommand(profile.getInstanceName(), controlNic.getIp4Address(), 3922)); + + // Update router template/scripts version + final GetDomRVersionCmd command = new GetDomRVersionCmd(); + command.setAccessDetail(NetworkElementCommand.ROUTER_IP, controlNic.getIp4Address()); + command.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); + cmds.addCommand("getDomRVersion", command); + + // Network usage command to create iptables rules + boolean forVpc = profile.getVirtualMachine().getVpcId() != null; + cmds.addCommand("networkUsage", new NetworkUsageCommand(controlNic.getIp4Address(), router.getHostName(), "create", forVpc)); + } + + protected void finalizeUserDataAndDhcpOnStart(Commands cmds, DomainRouterVO router, Provider provider, Long guestNetworkId) { + if (_networkMgr.isProviderSupportServiceInNetwork(guestNetworkId, Service.Dhcp, provider)) { + // Resend dhcp + s_logger.debug("Reapplying dhcp entries as a part of domR " + router + " start..."); + createDhcpEntryCommandsForVMs(router, cmds, guestNetworkId); + } + + if (_networkMgr.isProviderSupportServiceInNetwork(guestNetworkId, Service.UserData, provider)) { + // Resend user data + s_logger.debug("Reapplying vm data (userData and metaData) entries as a part of domR " + router + " start..."); + createVmDataCommandForVMs(router, cmds, guestNetworkId); + } + } + + protected void finalizeNetworkRulesForNetwork(Commands cmds, DomainRouterVO router, Provider provider, Long guestNetworkId) { + s_logger.debug("Resending ipAssoc, port forwarding, load balancing rules as a part of Virtual router start"); + + ArrayList publicIps = getPublicIpsToApply(router, provider, guestNetworkId); + + if (publicIps != null && !publicIps.isEmpty()) { + List vpns = new ArrayList(); + List pfRules = new ArrayList(); + List staticNatFirewallRules = new ArrayList(); + List staticNats = new ArrayList(); + List firewallRules = new ArrayList(); + + // Get information about all the rules (StaticNats and StaticNatRules; PFVPN to reapply on domR start) + for (PublicIpAddress ip : publicIps) { + if (_networkMgr.isProviderSupportServiceInNetwork(guestNetworkId, Service.PortForwarding, provider)) { + pfRules.addAll(_pfRulesDao.listForApplication(ip.getId())); + } + if (_networkMgr.isProviderSupportServiceInNetwork(guestNetworkId, Service.StaticNat, provider)) { + staticNatFirewallRules.addAll(_rulesDao.listByIpAndPurpose(ip.getId(), Purpose.StaticNat)); + } + if (_networkMgr.isProviderSupportServiceInNetwork(guestNetworkId, Service.Firewall, provider)) { + firewallRules.addAll(_rulesDao.listByIpAndPurpose(ip.getId(), Purpose.Firewall)); + } + + if (_networkMgr.isProviderSupportServiceInNetwork(guestNetworkId, Service.Vpn, provider)) { + RemoteAccessVpn vpn = _vpnDao.findById(ip.getId()); + if (vpn != null) { + vpns.add(vpn); + } + } + + if (_networkMgr.isProviderSupportServiceInNetwork(guestNetworkId, Service.StaticNat, provider)) { + if (ip.isOneToOneNat()) { + String dstIp = _networkMgr.getIpInNetwork(ip.getAssociatedWithVmId(), guestNetworkId); + StaticNatImpl staticNat = new StaticNatImpl(ip.getAccountId(), ip.getDomainId(), guestNetworkId, ip.getId(), dstIp, false); + staticNats.add(staticNat); + } + } + } + + // Re-apply static nats + s_logger.debug("Found " + staticNats.size() + " static nat(s) to apply as a part of domR " + router + " start."); + if (!staticNats.isEmpty()) { + createApplyStaticNatCommands(staticNats, router, cmds, guestNetworkId); + } + + // Re-apply firewall rules + s_logger.debug("Found " + staticNats.size() + " firewall rule(s) to apply as a part of domR " + router + " start."); + if (!firewallRules.isEmpty()) { + createFirewallRulesCommands(firewallRules, router, cmds, guestNetworkId); + } + + // Re-apply port forwarding rules + s_logger.debug("Found " + pfRules.size() + " port forwarding rule(s) to apply as a part of domR " + router + " start."); + if (!pfRules.isEmpty()) { + createApplyPortForwardingRulesCommands(pfRules, router, cmds, guestNetworkId); + } + + // Re-apply static nat rules + s_logger.debug("Found " + staticNatFirewallRules.size() + " static nat rule(s) to apply as a part of domR " + router + " start."); + if (!staticNatFirewallRules.isEmpty()) { + List staticNatRules = new ArrayList(); + for (FirewallRule rule : staticNatFirewallRules) { + staticNatRules.add(_rulesMgr.buildStaticNatRule(rule, false)); + } + createApplyStaticNatRulesCommands(staticNatRules, router, cmds, guestNetworkId); + } + + // Re-apply vpn rules + s_logger.debug("Found " + vpns.size() + " vpn(s) to apply as a part of domR " + router + " start."); + if (!vpns.isEmpty()) { + for (RemoteAccessVpn vpn : vpns) { + createApplyVpnCommands(vpn, router, cmds); + } + } + + List lbs = _loadBalancerDao.listByNetworkId(guestNetworkId); + List lbRules = new ArrayList(); + if (_networkMgr.isProviderSupportServiceInNetwork(guestNetworkId, Service.Lb, provider)) { + // Re-apply load balancing rules + for (LoadBalancerVO lb : lbs) { + List dstList = _lbMgr.getExistingDestinations(lb.getId()); + List policyList = _lbMgr.getStickinessPolicies(lb.getId()); + LoadBalancingRule loadBalancing = new LoadBalancingRule(lb, dstList, policyList); + lbRules.add(loadBalancing); + } + } + + s_logger.debug("Found " + lbRules.size() + " load balancing rule(s) to apply as a part of domR " + router + " start."); + if (!lbRules.isEmpty()) { + createApplyLoadBalancingRulesCommands(lbRules, router, cmds, guestNetworkId); + } + } + } + + protected void finalizeIpAssocForNetwork(Commands cmds, VirtualRouter router, Provider provider, + Long guestNetworkId, Map vlanMacAddress) { + + ArrayList publicIps = getPublicIpsToApply(router, provider, guestNetworkId); + + if (publicIps != null && !publicIps.isEmpty()) { + s_logger.debug("Found " + publicIps.size() + " ip(s) to apply as a part of domR " + router + " start."); + // Re-apply public ip addresses - should come before PF/LB/VPN + if (_networkMgr.isProviderSupportServiceInNetwork(guestNetworkId, Service.Firewall, provider)) { + createAssociateIPCommands(router, publicIps, cmds, 0); + } + } + } + + protected ArrayList getPublicIpsToApply(VirtualRouter router, Provider provider, + Long guestNetworkId, com.cloud.network.IpAddress.State... skipInStates) { + long ownerId = router.getAccountId(); + final List userIps = _networkMgr.listPublicIpsAssignedToGuestNtwk(ownerId, guestNetworkId, null); + List allPublicIps = new ArrayList(); + if (userIps != null && !userIps.isEmpty()) { + boolean addIp = true; + for (IPAddressVO userIp : userIps) { + if (skipInStates != null) { + for (IpAddress.State stateToSkip : skipInStates) { + if (userIp.getState() == stateToSkip) { + s_logger.debug("Skipping ip address " + userIp + " in state " + userIp.getState()); + addIp = false; + break; + } + } + } + + if (addIp) { + PublicIp publicIp = new PublicIp(userIp, _vlanDao.findById(userIp.getVlanId()), + NetUtils.createSequenceBasedMacAddress(userIp.getMacAddress())); + allPublicIps.add(publicIp); + } + } + } + + // Get public Ips that should be handled by router + Network network = _networkDao.findById(guestNetworkId); + Map> ipToServices = _networkMgr.getIpToServices(allPublicIps, false, false); + Map> providerToIpList = _networkMgr.getProviderToIpList(network, ipToServices); + // Only cover virtual router for now, if ELB use it this need to be modified + + ArrayList publicIps = providerToIpList.get(provider); + return publicIps; + } + + @Override + public boolean finalizeStart(VirtualMachineProfile profile, long hostId, Commands cmds, + ReservationContext context) { + DomainRouterVO router = profile.getVirtualMachine(); + + boolean result = true; + + Answer answer = cmds.getAnswer("checkSsh"); + if (answer != null && answer instanceof CheckSshAnswer) { + CheckSshAnswer sshAnswer = (CheckSshAnswer) answer; + if (sshAnswer == null || !sshAnswer.getResult()) { + s_logger.warn("Unable to ssh to the VM: " + sshAnswer.getDetails()); + result = false; + } + } else { + result = false; + } + if (result == false) { + return result; + } + + // Get guest networks info + List guestNetworks = new ArrayList(); + + List routerNics = _nicDao.listByVmId(profile.getId()); + for (Nic routerNic : routerNics) { + Network network = _networkMgr.getNetwork(routerNic.getNetworkId()); + if (network.getTrafficType() == TrafficType.Guest) { + guestNetworks.add(network); + } + } + + answer = cmds.getAnswer("getDomRVersion"); + if (answer != null && answer instanceof GetDomRVersionAnswer) { + GetDomRVersionAnswer versionAnswer = (GetDomRVersionAnswer) answer; + if (answer == null || !answer.getResult()) { + s_logger.warn("Unable to get the template/scripts version of router " + router.getInstanceName() + + " due to: " + versionAnswer.getDetails()); + result = false; + } else { + router.setTemplateVersion(versionAnswer.getTemplateVersion()); + router.setScriptsVersion(versionAnswer.getScriptsVersion()); + router = _routerDao.persist(router, guestNetworks); + } + } else { + result = false; + } + + return result; + } + + @Override + public void finalizeStop(VirtualMachineProfile profile, StopAnswer answer) { + if (answer != null) { + VMInstanceVO vm = profile.getVirtualMachine(); + DomainRouterVO domR = _routerDao.findById(vm.getId()); + processStopOrRebootAnswer(domR, answer); + } + } + + @Override + public void finalizeExpunge(DomainRouterVO vm) { + } + + @Override + public boolean startRemoteAccessVpn(Network network, RemoteAccessVpn vpn, List routers) + throws ResourceUnavailableException { + if (routers == null || routers.isEmpty()) { + s_logger.warn("Failed to start remote access VPN: no router found for account and zone"); + throw new ResourceUnavailableException("Failed to start remote access VPN: no router found for account and zone", + DataCenter.class, network.getDataCenterId()); + } + + for (VirtualRouter router : routers) { + if (router.getState() != State.Running) { + s_logger.warn("Failed to start remote access VPN: router not in right state " + router.getState()); + throw new ResourceUnavailableException("Failed to start remote access VPN: router not in right state " + + router.getState(), DataCenter.class, network.getDataCenterId()); + } + + Commands cmds = new Commands(OnError.Stop); + createApplyVpnCommands(vpn, router, cmds); + + try { + _agentMgr.send(router.getHostId(), cmds); + } catch (OperationTimedoutException e) { + s_logger.debug("Failed to start remote access VPN: ", e); + throw new AgentUnavailableException("Unable to send commands to virtual router ", router.getHostId(), e); + } + Answer answer = cmds.getAnswer("users"); + if (!answer.getResult()) { + s_logger.error("Unable to start vpn: unable add users to vpn in zone " + router.getDataCenterIdToDeployIn() + + " for account " + vpn.getAccountId() + " on domR: " + router.getInstanceName() + + " due to " + answer.getDetails()); + throw new ResourceUnavailableException("Unable to start vpn: Unable to add users to vpn in zone " + + router.getDataCenterIdToDeployIn() + " for account " + vpn.getAccountId() + " on domR: " + + router.getInstanceName() + " due to " + answer.getDetails(), DataCenter.class, router.getDataCenterIdToDeployIn()); + } + answer = cmds.getAnswer("startVpn"); + if (!answer.getResult()) { + s_logger.error("Unable to start vpn in zone " + router.getDataCenterIdToDeployIn() + " for account " + + vpn.getAccountId() + " on domR: " + router.getInstanceName() + " due to " + + answer.getDetails()); + throw new ResourceUnavailableException("Unable to start vpn in zone " + router.getDataCenterIdToDeployIn() + + " for account " + vpn.getAccountId() + " on domR: " + router.getInstanceName() + + " due to " + answer.getDetails(), DataCenter.class, router.getDataCenterIdToDeployIn()); + } + + } + return true; + } + + @Override + public boolean deleteRemoteAccessVpn(Network network, RemoteAccessVpn vpn, List routers) + throws ResourceUnavailableException { + if (routers == null || routers.isEmpty()) { + s_logger.warn("Failed to delete remote access VPN: no router found for account and zone"); + throw new ResourceUnavailableException("Failed to delete remote access VPN", DataCenter.class, network.getDataCenterId()); + } + + boolean result = true; + for (VirtualRouter router : routers) { + if (router.getState() == State.Running) { + Commands cmds = new Commands(OnError.Continue); + IpAddress ip = _networkMgr.getIp(vpn.getServerAddressId()); + + RemoteAccessVpnCfgCommand removeVpnCmd = new RemoteAccessVpnCfgCommand(false, ip.getAddress().addr(), + vpn.getLocalIp(), vpn.getIpRange(), vpn.getIpsecPresharedKey()); + removeVpnCmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); + removeVpnCmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(network.getId(), router.getId())); + removeVpnCmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); + + DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); + removeVpnCmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); + + cmds.addCommand(removeVpnCmd); + + result = result && sendCommandsToRouter(router, cmds); + } else if (router.getState() == State.Stopped) { + s_logger.debug("Router " + router + " is in Stopped state, not sending deleteRemoteAccessVpn command to it"); + continue; + } else { + s_logger.warn("Failed to delete remote access VPN: domR " + router + " is not in right state " + router.getState()); + throw new ResourceUnavailableException("Failed to delete remote access VPN: domR is not in right state " + + router.getState(), DataCenter.class, network.getDataCenterId()); + } + } + + return result; + } + + private DomainRouterVO start(DomainRouterVO router, User user, Account caller, Map params, DeploymentPlan planToDeploy) + throws StorageUnavailableException, InsufficientCapacityException, + ConcurrentOperationException, ResourceUnavailableException { + s_logger.debug("Starting router " + router); + if (_itMgr.start(router, params, user, caller, planToDeploy) != null) { + // We don't want the failure of VPN Connection affect the status of router, so we try to make connection +// only after router start successfully Long vpcId = router.getVpcId(); if (vpcId != null) { _s2sVpnMgr.reconnectDisconnectedVpnByVpc(vpcId); } - return _routerDao.findById(router.getId()); - } else { - return null; - } - } - - @Override - public DomainRouterVO stop(VirtualRouter router, boolean forced, User user, Account caller) throws ConcurrentOperationException, ResourceUnavailableException { - s_logger.debug("Stopping router " + router); - try { - if (_itMgr.advanceStop((DomainRouterVO) router, forced, user, caller)) { - return _routerDao.findById(router.getId()); - } else { - return null; - } - } catch (OperationTimedoutException e) { - throw new CloudRuntimeException("Unable to stop " + router, e); - } - } - - @Override - public boolean applyDhcpEntry(Network network, final NicProfile nic, VirtualMachineProfile profile, - DeployDestination dest, List routers) - throws ResourceUnavailableException { - _userVmDao.loadDetails((UserVmVO) profile.getVirtualMachine()); - - final VirtualMachineProfile updatedProfile = profile; - final boolean isZoneBasic = (dest.getDataCenter().getNetworkType() == NetworkType.Basic); - final Long podId = isZoneBasic ? dest.getPod().getId() : null; - - boolean podLevelException = false; - //for user vm in Basic zone we should try to re-deploy vm in a diff pod if it fails to deploy in original pod; so throwing exception with Pod scope - if (isZoneBasic && podId != null && updatedProfile.getVirtualMachine().getType() == VirtualMachine.Type.User - && network.getTrafficType() == TrafficType.Guest && network.getGuestType() == Network.GuestType.Shared) { - podLevelException = true; - } - - return applyRules(network, routers, "dhcp entry", podLevelException, podId, true, new RuleApplier() { - @Override - public boolean execute(Network network, VirtualRouter router) throws ResourceUnavailableException { - //for basic zone, send dhcp/dns information to all routers in the basic network only when _dnsBasicZoneUpdates is set to "all" value - Commands cmds = new Commands(OnError.Stop); - if (!(isZoneBasic && router.getPodIdToDeployIn().longValue() != podId.longValue() && _dnsBasicZoneUpdates.equalsIgnoreCase("pod"))) { - NicVO nicVo = _nicDao.findById(nic.getId()); - createDhcpEntryCommand(router, updatedProfile.getVirtualMachine(), nicVo, cmds); - return sendCommandsToRouter(router, cmds); - } - return true; - } - }); - } - - private String findDefaultDnsIp(long userVmId) { - NicVO defaultNic = _nicDao.findDefaultNicForVM(userVmId); - - //check if DNS provider is the domR - if (!_networkMgr.isProviderSupportServiceInNetwork(defaultNic.getNetworkId(), Service.Dns, Provider.VirtualRouter)) { - return null; - } - - NetworkOfferingVO offering = _networkOfferingDao.findById(_networkDao.findById(defaultNic.getNetworkId()).getNetworkOfferingId()); - if (offering.getRedundantRouter()) { - return findGatewayIp(userVmId); - } - - DataCenter dc = _dcDao.findById(_networkMgr.getNetwork(defaultNic.getNetworkId()).getDataCenterId()); - boolean isZoneBasic = (dc.getNetworkType() == NetworkType.Basic); - - //find domR's nic in the network - NicVO domrDefaultNic; - if (isZoneBasic){ - domrDefaultNic = _nicDao.findByNetworkIdTypeAndGateway(defaultNic.getNetworkId(), VirtualMachine.Type.DomainRouter, defaultNic.getGateway()); - } - else{ - domrDefaultNic = _nicDao.findByNetworkIdAndType(defaultNic.getNetworkId(), VirtualMachine.Type.DomainRouter); - } - - return domrDefaultNic.getIp4Address(); - } - - private String findGatewayIp(long userVmId) { - NicVO defaultNic = _nicDao.findDefaultNicForVM(userVmId); - return defaultNic.getGateway(); - } - - @Override - public boolean applyUserData(Network network, final NicProfile nic, VirtualMachineProfile profile, DeployDestination dest, List routers) - throws ResourceUnavailableException { - _userVmDao.loadDetails((UserVmVO) profile.getVirtualMachine()); - - final VirtualMachineProfile updatedProfile = profile; - final boolean isZoneBasic = (dest.getDataCenter().getNetworkType() == NetworkType.Basic); - final Long podId = isZoneBasic ? dest.getPod().getId() : null; - - boolean podLevelException = false; - //for user vm in Basic zone we should try to re-deploy vm in a diff pod if it fails to deploy in original pod; so throwing exception with Pod scope - if (isZoneBasic && podId != null && updatedProfile.getVirtualMachine().getType() == VirtualMachine.Type.User - && network.getTrafficType() == TrafficType.Guest && network.getGuestType() == Network.GuestType.Shared) { - podLevelException = true; - } - - return applyRules(network, routers, "userdata and password entry", podLevelException, podId, false, new RuleApplier() { - @Override - public boolean execute(Network network, VirtualRouter router) throws ResourceUnavailableException { - //for basic zone, send vm data/password information only to the router in the same pod - Commands cmds = new Commands(OnError.Stop); - if (!(isZoneBasic && router.getPodIdToDeployIn().longValue() != podId.longValue())) { - NicVO nicVo = _nicDao.findById(nic.getId()); - createPasswordCommand(router, updatedProfile, nicVo, cmds); - createVmDataCommand(router, updatedProfile.getVirtualMachine(), nicVo, updatedProfile.getVirtualMachine().getDetail("SSH.PublicKey"), cmds); - return sendCommandsToRouter(router, cmds); - } - return true; - } - }); - } - - @Override - public DomainRouterVO persist(DomainRouterVO router) { - DomainRouterVO virtualRouter = _routerDao.persist(router); - return virtualRouter; - } - - @Override - //FIXME add partial success and STOP state support - public String[] applyVpnUsers(Network network, List users, List routers) throws ResourceUnavailableException { - if (routers == null || routers.isEmpty()) { - s_logger.warn("Failed to add/remove VPN users: no router found for account and zone"); - throw new ResourceUnavailableException("Unable to assign ip addresses, domR doesn't exist for network " + - network.getId(), DataCenter.class, network.getDataCenterId()); - } - - boolean agentResults = true; - - for (DomainRouterVO router : routers) { - if (router.getState() != State.Running) { - s_logger.warn("Failed to add/remove VPN users: router not in running state"); - throw new ResourceUnavailableException("Unable to assign ip addresses, domR is not in right state " + - router.getState(), DataCenter.class, network.getDataCenterId()); - } - - Commands cmds = new Commands(OnError.Continue); - List addUsers = new ArrayList(); - List removeUsers = new ArrayList(); - for (VpnUser user : users) { - if (user.getState() == VpnUser.State.Add || user.getState() == VpnUser.State.Active) { - addUsers.add(user); - } else if (user.getState() == VpnUser.State.Revoke) { - removeUsers.add(user); - } - } - - VpnUsersCfgCommand cmd = new VpnUsersCfgCommand(addUsers, removeUsers); - cmd.setAccessDetail(NetworkElementCommand.ACCOUNT_ID, String.valueOf(router.getAccountId())); - cmd.setAccessDetail(NetworkElementCommand.GUEST_NETWORK_CIDR, network.getCidr()); - cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); - cmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(network.getId(), router.getId())); - cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); - DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); - cmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); - - cmds.addCommand(cmd); - - - // Currently we receive just one answer from the agent. In the future we have to parse individual answers and set - // results accordingly - boolean agentResult = sendCommandsToRouter(router, cmds); - agentResults = agentResults && agentResult; - } - - String[] result = new String[users.size()]; - for (int i = 0; i < result.length; i++) { - if (agentResults) { - result[i] = null; - } else { - result[i] = String.valueOf(agentResults); - } - } - - return result; - } - - @Override - public DomainRouterVO findById(long id) { - return _routerDao.findById(id); - } - - @Override - public DomainRouterVO findByName(String name) { - if (!VirtualMachineName.isValidRouterName(name)) { - return null; - } - - return _routerDao.findById(VirtualMachineName.getRouterId(name)); - } - - @Override @ActionEvent(eventType = EventTypes.EVENT_ROUTER_START, eventDescription = "starting router Vm", async = true) - public VirtualRouter startRouter(long id) throws ResourceUnavailableException, InsufficientCapacityException, ConcurrentOperationException{ - return startRouter(id, true); - } - - @Override - public VirtualRouter startRouter(long routerId, boolean reprogramNetwork) throws ResourceUnavailableException, - InsufficientCapacityException, ConcurrentOperationException { - Account caller = UserContext.current().getCaller(); - User callerUser = _accountMgr.getActiveUser(UserContext.current().getCallerUserId()); - - // verify parameters - DomainRouterVO router = _routerDao.findById(routerId); - if (router == null) { - throw new InvalidParameterValueException("Unable to find router by id", null); - } - _accountMgr.checkAccess(caller, null, true, router); - - Account owner = _accountMgr.getAccount(router.getAccountId()); - - // Check if all networks are implemented for the domR; if not - implement them - DataCenter dc = _dcDao.findById(router.getDataCenterIdToDeployIn()); - HostPodVO pod = null; - if (router.getPodIdToDeployIn() != null) { - pod = _podDao.findById(router.getPodIdToDeployIn()); - } - DeployDestination dest = new DeployDestination(dc, pod, null, null); - - ReservationContext context = new ReservationContextImpl(null, null, callerUser, owner); - - List nics = _nicDao.listByVmId(routerId); - - for (NicVO nic : nics) { - if (!_networkMgr.startNetwork(nic.getNetworkId(), dest, context)) { - s_logger.warn("Failed to start network id=" + nic.getNetworkId() + " as a part of domR start"); - throw new CloudRuntimeException("Failed to start network id=" + nic.getNetworkId() + " as a part of domR start"); - } - } - - UserVO user = _userDao.findById(UserContext.current().getCallerUserId()); - Map params = new HashMap(); - if (reprogramNetwork) { - params.put(Param.ReProgramGuestNetworks, true); - } else { - params.put(Param.ReProgramGuestNetworks, false); - } - VirtualRouter virtualRouter = startVirtualRouter(router, user, caller, params); - if(virtualRouter == null){ - throw new CloudRuntimeException("Failed to start router with id " + routerId); - } - return virtualRouter; - } - - private void createAssociateIPCommands(final VirtualRouter router, final List ips, Commands cmds, long vmId) { - - // Ensure that in multiple vlans case we first send all ip addresses of vlan1, then all ip addresses of vlan2, etc.. - Map> vlanIpMap = new HashMap>(); - for (final PublicIpAddress ipAddress : ips) { - String vlanTag = ipAddress.getVlanTag(); - ArrayList ipList = vlanIpMap.get(vlanTag); - if (ipList == null) { - ipList = new ArrayList(); - } - //domR doesn't support release for sourceNat IP address; so reset the state - if (ipAddress.isSourceNat() && ipAddress.getState() == IpAddress.State.Releasing) { - ipAddress.setState(IpAddress.State.Allocated); - } - ipList.add(ipAddress); - vlanIpMap.put(vlanTag, ipList); - } - - for (Map.Entry> vlanAndIp : vlanIpMap.entrySet()) { - List ipAddrList = vlanAndIp.getValue(); - // Source nat ip address should always be sent first - Collections.sort(ipAddrList, new Comparator() { - @Override - public int compare(PublicIpAddress o1, PublicIpAddress o2) { - boolean s1 = o1.isSourceNat(); - boolean s2 = o2.isSourceNat(); - return (s1 ^ s2) ? ((s1 ^ true) ? 1 : -1) : 0; - } - }); - - // Get network rate - required for IpAssoc - Integer networkRate = _networkMgr.getNetworkRate(ipAddrList.get(0).getNetworkId(), router.getId()); - Network network = _networkMgr.getNetwork(ipAddrList.get(0).getNetworkId()); - - IpAddressTO[] ipsToSend = new IpAddressTO[ipAddrList.size()]; - int i = 0; - boolean firstIP = true; - - for (final PublicIpAddress ipAddr : ipAddrList) { - - boolean add = (ipAddr.getState() == IpAddress.State.Releasing ? false : true); - boolean sourceNat = ipAddr.isSourceNat(); - /* enable sourceNAT for the first ip of the public interface */ - if (firstIP) { - sourceNat = true; - } - String vlanId = ipAddr.getVlanTag(); - String vlanGateway = ipAddr.getGateway(); - String vlanNetmask = ipAddr.getNetmask(); - String vifMacAddress = ipAddr.getMacAddress(); - - String vmGuestAddress = null; - - IpAddressTO ip = new IpAddressTO(ipAddr.getAccountId(), ipAddr.getAddress().addr(), add, firstIP, - sourceNat, vlanId, vlanGateway, vlanNetmask, vifMacAddress, vmGuestAddress, networkRate, ipAddr.isOneToOneNat()); - - ip.setTrafficType(network.getTrafficType()); - ip.setNetworkName(_networkMgr.getNetworkTag(router.getHypervisorType(), network)); - ipsToSend[i++] = ip; - /* send the firstIP = true for the first Add, this is to create primary on interface*/ - if (!firstIP || add) { - firstIP = false; - } - } - IpAssocCommand cmd = new IpAssocCommand(ipsToSend); - cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); - cmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(ipAddrList.get(0).getNetworkId(), router.getId())); - cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); - DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); - cmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); - - cmds.addCommand("IPAssocCommand", cmd); - } - } - - private void createApplyPortForwardingRulesCommands(List rules, VirtualRouter router, Commands cmds, long guestNetworkId) { - List rulesTO = null; - if (rules != null) { - rulesTO = new ArrayList(); - for (PortForwardingRule rule : rules) { - IpAddress sourceIp = _networkMgr.getIp(rule.getSourceIpAddressId()); - PortForwardingRuleTO ruleTO = new PortForwardingRuleTO(rule, null, sourceIp.getAddress().addr()); - rulesTO.add(ruleTO); - } - } - - SetPortForwardingRulesCommand cmd = null; - - if (router.getVpcId() != null) { - cmd = new SetPortForwardingRulesVpcCommand(rulesTO); - } else { - cmd = new SetPortForwardingRulesCommand(rulesTO); - } - - cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); - cmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(guestNetworkId, router.getId())); - cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); - DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); - cmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); - - cmds.addCommand(cmd); - } - - private void createApplyStaticNatRulesCommands(List rules, VirtualRouter router, Commands cmds, long guestNetworkId) { - List rulesTO = null; - if (rules != null) { - rulesTO = new ArrayList(); - for (StaticNatRule rule : rules) { - IpAddress sourceIp = _networkMgr.getIp(rule.getSourceIpAddressId()); - StaticNatRuleTO ruleTO = new StaticNatRuleTO(rule, null, sourceIp.getAddress().addr(), rule.getDestIpAddress()); - rulesTO.add(ruleTO); - } - } - - SetStaticNatRulesCommand cmd = new SetStaticNatRulesCommand(rulesTO, router.getVpcId()); - cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); - cmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(guestNetworkId, router.getId())); - cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); - DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); - cmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); - cmds.addCommand(cmd); - } - - private void createApplyLoadBalancingRulesCommands(List rules, VirtualRouter router, Commands cmds, long guestNetworkId) { - - LoadBalancerTO[] lbs = new LoadBalancerTO[rules.size()]; - int i = 0; - for (LoadBalancingRule rule : rules) { - boolean revoked = (rule.getState().equals(FirewallRule.State.Revoke)); - String protocol = rule.getProtocol(); - String algorithm = rule.getAlgorithm(); - - String srcIp = _networkMgr.getIp(rule.getSourceIpAddressId()).getAddress().addr(); - int srcPort = rule.getSourcePortStart(); - List destinations = rule.getDestinations(); - List stickinessPolicies = rule.getStickinessPolicies(); - LoadBalancerTO lb = new LoadBalancerTO(rule.getId(), srcIp, srcPort, protocol, algorithm, revoked, false, destinations, stickinessPolicies); - lbs[i++] = lb; - } - String routerPublicIp = null; - - if (router instanceof DomainRouterVO) { - DomainRouterVO domr = _routerDao.findById(router.getId()); - routerPublicIp = domr.getPublicIpAddress(); - } - - Network guestNetwork = _networkMgr.getNetwork(guestNetworkId); - Nic nic = _nicDao.findByInstanceIdAndNetworkId(guestNetwork.getId(), router.getId()); - NicProfile nicProfile = new NicProfile(nic, guestNetwork, nic.getBroadcastUri(), nic.getIsolationUri(), - _networkMgr.getNetworkRate(guestNetwork.getId(), router.getId()), - _networkMgr.isSecurityGroupSupportedInNetwork(guestNetwork), - _networkMgr.getNetworkTag(router.getHypervisorType(), guestNetwork)); - - LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(lbs,routerPublicIp, - getRouterIpInNetwork(guestNetworkId, router.getId()),router.getPrivateIpAddress(), - _itMgr.toNicTO(nicProfile, router.getHypervisorType()), router.getVpcId()); - - cmd.lbStatsVisibility = _configDao.getValue(Config.NetworkLBHaproxyStatsVisbility.key()); - cmd.lbStatsUri = _configDao.getValue(Config.NetworkLBHaproxyStatsUri.key()); - cmd.lbStatsAuth = _configDao.getValue(Config.NetworkLBHaproxyStatsAuth.key()); - cmd.lbStatsPort = _configDao.getValue(Config.NetworkLBHaproxyStatsPort.key()); - - - cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); - cmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(guestNetworkId, router.getId())); - cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); - DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); - cmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); - cmds.addCommand(cmd); - - } - - private void createApplyVpnCommands(RemoteAccessVpn vpn, VirtualRouter router, Commands cmds) { - List vpnUsers = _vpnUsersDao.listByAccount(vpn.getAccountId()); - List addUsers = new ArrayList(); - List removeUsers = new ArrayList(); - for (VpnUser user : vpnUsers) { - if (user.getState() == VpnUser.State.Add) { - addUsers.add(user); - } else if (user.getState() == VpnUser.State.Revoke) { - removeUsers.add(user); - } - } - - VpnUsersCfgCommand addUsersCmd = new VpnUsersCfgCommand(addUsers, removeUsers); - addUsersCmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); - addUsersCmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(vpn.getNetworkId(), router.getId())); - addUsersCmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); - - IpAddress ip = _networkMgr.getIp(vpn.getServerAddressId()); - - RemoteAccessVpnCfgCommand startVpnCmd = new RemoteAccessVpnCfgCommand(true, ip.getAddress().addr(), - vpn.getLocalIp(), vpn.getIpRange(), vpn.getIpsecPresharedKey()); - startVpnCmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); - startVpnCmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(vpn.getNetworkId(), router.getId())); - startVpnCmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); - DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); - startVpnCmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); - - cmds.addCommand("users", addUsersCmd); - cmds.addCommand("startVpn", startVpnCmd); - } - - private void createPasswordCommand(VirtualRouter router, VirtualMachineProfile profile, NicVO nic, Commands cmds) { - String password = (String) profile.getParameter(VirtualMachineProfile.Param.VmPassword); - DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); - - // password should be set only on default network element - if (password != null && nic.isDefaultNic()) { - final String encodedPassword = PasswordGenerator.rot13(password); - SavePasswordCommand cmd = new SavePasswordCommand(encodedPassword, nic.getIp4Address(), profile.getVirtualMachine().getHostName()); - cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); - cmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(nic.getNetworkId(), router.getId())); - cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); - cmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); - - cmds.addCommand("password", cmd); - } - - } - - private void createVmDataCommand(VirtualRouter router, UserVm vm, NicVO nic, String publicKey, Commands cmds) { - String serviceOffering = _serviceOfferingDao.findByIdIncludingRemoved(vm.getServiceOfferingId()).getDisplayText(); - String zoneName = _dcDao.findById(router.getDataCenterIdToDeployIn()).getName(); - cmds.addCommand("vmdata", - generateVmDataCommand(router, nic.getIp4Address(), vm.getUserData(), serviceOffering, zoneName, nic.getIp4Address(), - vm.getHostName(), vm.getInstanceName(), vm.getId(), publicKey, nic.getNetworkId())); - - } - - private void createVmDataCommandForVMs(DomainRouterVO router, Commands cmds, long guestNetworkId) { - List vms = _userVmDao.listByNetworkIdAndStates(guestNetworkId, State.Running, State.Migrating, State.Stopping); - DataCenterVO dc = _dcDao.findById(router.getDataCenterIdToDeployIn()); - for (UserVmVO vm : vms) { - boolean createVmData = true; - if (dc.getNetworkType() == NetworkType.Basic && router.getPodIdToDeployIn().longValue() != vm.getPodIdToDeployIn().longValue()) { - createVmData = false; - } - - if (createVmData) { - NicVO nic = _nicDao.findByInstanceIdAndNetworkId(guestNetworkId, vm.getId()); - if (nic != null) { - s_logger.debug("Creating user data entry for vm " + vm + " on domR " + router); - createVmDataCommand(router, vm, nic, null, cmds); - } - } - } - } - - private void createDhcpEntryCommand(VirtualRouter router, UserVm vm, NicVO nic, Commands cmds) { - DhcpEntryCommand dhcpCommand = new DhcpEntryCommand(nic.getMacAddress(), nic.getIp4Address(), vm.getHostName()); - DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); - String gatewayIp = findGatewayIp(vm.getId()); - if (!gatewayIp.equals(nic.getGateway())) { - GuestOSVO guestOS = _guestOSDao.findById(vm.getGuestOSId()); - // Don't set dhcp:router option for non-default nic on CentOS/RHEL, because they would set routing on wrong interface - // This is tricky, we may need to update this when we have more information on various OS's behavior - if (guestOS.getDisplayName().startsWith("CentOS") || guestOS.getDisplayName().startsWith("Red Hat Enterprise")) { - gatewayIp = "0.0.0.0"; - } - } - dhcpCommand.setDefaultRouter(gatewayIp); - dhcpCommand.setDefaultDns(findDefaultDnsIp(vm.getId())); - - dhcpCommand.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); - dhcpCommand.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); - dhcpCommand.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(nic.getNetworkId(), router.getId())); - dhcpCommand.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); - - cmds.addCommand("dhcp", dhcpCommand); - } - - private void createDhcpEntryCommandsForVMs(DomainRouterVO router, Commands cmds, long guestNetworkId) { - List vms = _userVmDao.listByNetworkIdAndStates(guestNetworkId, State.Running, State.Migrating, State.Stopping); - DataCenterVO dc = _dcDao.findById(router.getDataCenterIdToDeployIn()); - for (UserVmVO vm : vms) { - boolean createDhcp = true; - if (dc.getNetworkType() == NetworkType.Basic && router.getPodIdToDeployIn().longValue() != vm.getPodIdToDeployIn().longValue() - && _dnsBasicZoneUpdates.equalsIgnoreCase("pod")) { - createDhcp = false; - } - if (createDhcp) { - NicVO nic = _nicDao.findByInstanceIdAndNetworkId(guestNetworkId, vm.getId()); - if (nic != null) { - s_logger.debug("Creating dhcp entry for vm " + vm + " on domR " + router + "."); - createDhcpEntryCommand(router, vm, nic, cmds); - } - } - } - } - - protected boolean sendCommandsToRouter(final VirtualRouter router, Commands cmds) throws AgentUnavailableException { - Answer[] answers = null; - try { - answers = _agentMgr.send(router.getHostId(), cmds); - } catch (OperationTimedoutException e) { - s_logger.warn("Timed Out", e); - throw new AgentUnavailableException("Unable to send commands to virtual router ", router.getHostId(), e); - } - - if (answers == null) { - return false; - } - - if (answers.length != cmds.size()) { - return false; - } - - // FIXME: Have to return state for individual command in the future - boolean result = true; - if (answers.length > 0) { - for (Answer answer : answers) { - if (!answer.getResult()) { - result = false; - break; - } - } - } - return result; - } - - protected void handleSingleWorkingRedundantRouter(List connectedRouters, List disconnectedRouters, String reason) throws ResourceUnavailableException - { - if (connectedRouters.isEmpty() || disconnectedRouters.isEmpty()) { - return; - } - if (connectedRouters.size() != 1 || disconnectedRouters.size() != 1) { - s_logger.warn("How many redundant routers do we have?? "); - return; - } - if (!connectedRouters.get(0).getIsRedundantRouter()) { - throw new ResourceUnavailableException("Who is calling this with non-redundant router or non-domain router?", - DataCenter.class, connectedRouters.get(0).getDataCenterIdToDeployIn()); - } - if (!disconnectedRouters.get(0).getIsRedundantRouter()) { - throw new ResourceUnavailableException("Who is calling this with non-redundant router or non-domain router?", - DataCenter.class, disconnectedRouters.get(0).getDataCenterIdToDeployIn()); - } - - DomainRouterVO connectedRouter = (DomainRouterVO)connectedRouters.get(0); - DomainRouterVO disconnectedRouter = (DomainRouterVO)disconnectedRouters.get(0); - - if (s_logger.isDebugEnabled()) { - s_logger.debug("About to stop the router " + disconnectedRouter.getInstanceName() + " due to: " + reason); - } - String title = "Virtual router " + disconnectedRouter.getInstanceName() + " would be stopped after connecting back, due to " + reason; - String context = "Virtual router (name: " + disconnectedRouter.getInstanceName() + ", id: " + disconnectedRouter.getId() + ") would be stopped after connecting back, due to: " + reason; - _alertMgr.sendAlert(AlertManager.ALERT_TYPE_DOMAIN_ROUTER, - disconnectedRouter.getDataCenterIdToDeployIn(), disconnectedRouter.getPodIdToDeployIn(), title, context); - disconnectedRouter.setStopPending(true); - disconnectedRouter = _routerDao.persist(disconnectedRouter); - - int connRouterPR = getRealPriority(connectedRouter); - int disconnRouterPR = getRealPriority(disconnectedRouter); - if (connRouterPR < disconnRouterPR) { - //connRouterPR < disconnRouterPR, they won't equal at anytime - if (!connectedRouter.getIsPriorityBumpUp()) { - final BumpUpPriorityCommand command = new BumpUpPriorityCommand(); - command.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(connectedRouter.getId())); - command.setAccessDetail(NetworkElementCommand.ROUTER_NAME, connectedRouter.getInstanceName()); - final Answer answer = _agentMgr.easySend(connectedRouter.getHostId(), command); - if (!answer.getResult()) { - s_logger.error("Failed to bump up " + connectedRouter.getInstanceName() + "'s priority! " + answer.getDetails()); - } - } else { - String t = "Can't bump up virtual router " + connectedRouter.getInstanceName() + "'s priority due to it's already bumped up!"; - _alertMgr.sendAlert(AlertManager.ALERT_TYPE_DOMAIN_ROUTER, - connectedRouter.getDataCenterIdToDeployIn(), connectedRouter.getPodIdToDeployIn(), t, t); - } - } - } - - @Override - public boolean associatePublicIP(Network network, final List ipAddress, List routers) - throws ResourceUnavailableException { - if (ipAddress == null || ipAddress.isEmpty()) { - s_logger.debug("No ip association rules to be applied for network " + network.getId()); - return true; - } - return applyRules(network, routers, "ip association", false, null, false, new RuleApplier() { - @Override - public boolean execute(Network network, VirtualRouter router) throws ResourceUnavailableException { - Commands cmds = new Commands(OnError.Continue); - createAssociateIPCommands(router, ipAddress, cmds, 0); - return sendCommandsToRouter(router, cmds); - } - }); - } - - @Override - public boolean applyFirewallRules(Network network, final List rules, List routers) throws ResourceUnavailableException { - if (rules == null || rules.isEmpty()) { - s_logger.debug("No firewall rules to be applied for network " + network.getId()); - return true; - } - return applyRules(network, routers, "firewall rules", false, null, false, new RuleApplier() { - @Override - public boolean execute(Network network, VirtualRouter router) throws ResourceUnavailableException { - if (rules.get(0).getPurpose() == Purpose.LoadBalancing) { - // for load balancer we have to resend all lb rules for the network - List lbs = _loadBalancerDao.listByNetworkId(network.getId()); - List lbRules = new ArrayList(); - for (LoadBalancerVO lb : lbs) { - List dstList = _lbMgr.getExistingDestinations(lb.getId()); - List policyList = _lbMgr.getStickinessPolicies(lb.getId()); - LoadBalancingRule loadBalancing = new LoadBalancingRule(lb, dstList,policyList); - lbRules.add(loadBalancing); - } - return sendLBRules(router, lbRules, network.getId()); - } else if (rules.get(0).getPurpose() == Purpose.PortForwarding) { - return sendPortForwardingRules(router, (List) rules, network.getId()); - } else if (rules.get(0).getPurpose() == Purpose.StaticNat) { - return sendStaticNatRules(router, (List) rules, network.getId()); - } else if (rules.get(0).getPurpose() == Purpose.Firewall) { - return sendFirewallRules(router, (List) rules, network.getId()); - } else { - s_logger.warn("Unable to apply rules of purpose: " + rules.get(0).getPurpose()); - return false; - } - } - }); - } - - protected boolean sendLBRules(VirtualRouter router, List rules, long guestNetworkId) throws ResourceUnavailableException { - Commands cmds = new Commands(OnError.Continue); - createApplyLoadBalancingRulesCommands(rules, router, cmds, guestNetworkId); - return sendCommandsToRouter(router, cmds); - } - - protected boolean sendPortForwardingRules(VirtualRouter router, List rules, long guestNetworkId) throws ResourceUnavailableException { - Commands cmds = new Commands(OnError.Continue); - createApplyPortForwardingRulesCommands(rules, router, cmds, guestNetworkId); - return sendCommandsToRouter(router, cmds); - } - - protected boolean sendStaticNatRules(VirtualRouter router, List rules, long guestNetworkId) throws ResourceUnavailableException { - Commands cmds = new Commands(OnError.Continue); - createApplyStaticNatRulesCommands(rules, router, cmds, guestNetworkId); - return sendCommandsToRouter(router, cmds); - } - - @Override - public List getRoutersForNetwork(long networkId) { - List routers = _routerDao.findByNetwork(networkId); - List vrs = new ArrayList(routers.size()); - for (DomainRouterVO router : routers) { - vrs.add(router); - } - return vrs; - } - - private void createFirewallRulesCommands(List rules, VirtualRouter router, Commands cmds, long guestNetworkId) { - List rulesTO = null; - if (rules != null) { - rulesTO = new ArrayList(); - for (FirewallRule rule : rules) { - IpAddress sourceIp = _networkMgr.getIp(rule.getSourceIpAddressId()); - FirewallRuleTO ruleTO = new FirewallRuleTO(rule, null, sourceIp.getAddress().addr()); - rulesTO.add(ruleTO); - } - } - - SetFirewallRulesCommand cmd = new SetFirewallRulesCommand(rulesTO); - cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); - cmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(guestNetworkId, router.getId())); - cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); - DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); - cmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); - cmds.addCommand(cmd); - } - - - protected boolean sendFirewallRules(VirtualRouter router, List rules, long guestNetworkId) throws ResourceUnavailableException { - Commands cmds = new Commands(OnError.Continue); - createFirewallRulesCommands(rules, router, cmds, guestNetworkId); - return sendCommandsToRouter(router, cmds); - } - - @Override - public String getDnsBasicZoneUpdate() { - return _dnsBasicZoneUpdates; - } - - protected interface RuleApplier { - boolean execute(Network network, VirtualRouter router) throws ResourceUnavailableException; - } - - protected boolean applyRules(Network network, List routers, String typeString, - boolean isPodLevelException, Long podId, boolean failWhenDisconnect, RuleApplier applier) throws ResourceUnavailableException { - if (routers == null || routers.isEmpty()) { - s_logger.warn("Unable to apply " + typeString + ", virtual router doesn't exist in the network " + network.getId()); - throw new ResourceUnavailableException("Unable to apply " + typeString , DataCenter.class, network.getDataCenterId()); - } - - DataCenter dc = _dcDao.findById(network.getDataCenterId()); - boolean isZoneBasic = (dc.getNetworkType() == NetworkType.Basic); - - // isPodLevelException and podId is only used for basic zone - assert !((!isZoneBasic && isPodLevelException) || (isZoneBasic && isPodLevelException && podId == null)); - - List connectedRouters = new ArrayList(); - List disconnectedRouters = new ArrayList(); - boolean result = true; - String msg = "Unable to apply " + typeString + " on disconnected router "; - for (VirtualRouter router : routers) { - if (router.getState() == State.Running) { - s_logger.debug("Applying " + typeString + " in network " + network); - - if (router.isStopPending()) { - if (_hostDao.findById(router.getHostId()).getStatus() == Status.Up) { - throw new ResourceUnavailableException("Unable to process due to the stop pending router " + - router.getInstanceName() + " haven't been stopped after it's host coming back!", - DataCenter.class, router.getDataCenterIdToDeployIn()); - } - s_logger.debug("Router " + router.getInstanceName() + " is stop pending, so not sending apply " + - typeString + " commands to the backend"); - continue; - } - try { - result = applier.execute(network, router); - connectedRouters.add(router); - } catch (AgentUnavailableException e) { - s_logger.warn(msg + router.getInstanceName(), e); - disconnectedRouters.add(router); - } - - //If rules fail to apply on one domR and not due to disconnection, no need to proceed with the rest - if (!result) { - if (isZoneBasic && isPodLevelException) { - throw new ResourceUnavailableException("Unable to apply " + typeString + " on router ", Pod.class, podId); - } - throw new ResourceUnavailableException("Unable to apply " + typeString + " on router ", DataCenter.class, - router.getDataCenterIdToDeployIn()); - } - - } else if (router.getState() == State.Stopped || router.getState() == State.Stopping) { - s_logger.debug("Router " + router.getInstanceName() + " is in " + router.getState() + - ", so not sending apply " + typeString + " commands to the backend"); - } else { - s_logger.warn("Unable to apply " + typeString +", virtual router is not in the right state " + router.getState()); - if (isZoneBasic && isPodLevelException) { - throw new ResourceUnavailableException("Unable to apply " + typeString + - ", virtual router is not in the right state", Pod.class, podId); - } - throw new ResourceUnavailableException("Unable to apply " + typeString + - ", virtual router is not in the right state", DataCenter.class, router.getDataCenterIdToDeployIn()); - } - } - - if (!connectedRouters.isEmpty()) { - if (!isZoneBasic && !disconnectedRouters.isEmpty() && disconnectedRouters.get(0).getIsRedundantRouter()) { - // These disconnected redundant virtual routers are out of sync now, stop them for synchronization - handleSingleWorkingRedundantRouter(connectedRouters, disconnectedRouters, msg); - } - } else if (!disconnectedRouters.isEmpty()) { - for (VirtualRouter router : disconnectedRouters) { - if (s_logger.isDebugEnabled()) { - s_logger.debug(msg + router.getInstanceName() + "(" + router.getId() + ")"); - } - } - if (isZoneBasic && isPodLevelException) { - throw new ResourceUnavailableException(msg, Pod.class, podId); - } - throw new ResourceUnavailableException(msg, DataCenter.class, disconnectedRouters.get(0).getDataCenterIdToDeployIn()); - } - - result = true; - if (failWhenDisconnect) { - result = !connectedRouters.isEmpty(); - } - return result; - } - - @Override - public boolean applyStaticNats(Network network, final List rules, List routers) throws ResourceUnavailableException { - if (rules == null || rules.isEmpty()) { - s_logger.debug("No static nat rules to be applied for network " + network.getId()); - return true; - } - return applyRules(network, routers, "static nat rules", false, null, false, new RuleApplier() { - @Override - public boolean execute(Network network, VirtualRouter router) throws ResourceUnavailableException { - return applyStaticNat(router, rules, network.getId()); - } - }); - } - - - protected boolean applyStaticNat(VirtualRouter router, List rules, long guestNetworkId) throws ResourceUnavailableException { - Commands cmds = new Commands(OnError.Continue); - createApplyStaticNatCommands(rules, router, cmds, guestNetworkId); - return sendCommandsToRouter(router, cmds); - } - - private void createApplyStaticNatCommands(List rules, VirtualRouter router, Commands cmds, - long guestNetworkId) { - List rulesTO = null; - if (rules != null) { - rulesTO = new ArrayList(); - for (StaticNat rule : rules) { - IpAddress sourceIp = _networkMgr.getIp(rule.getSourceIpAddressId()); - StaticNatRuleTO ruleTO = new StaticNatRuleTO(0, sourceIp.getAddress().addr(), null, - null, rule.getDestIpAddress(), null, null, null, rule.isForRevoke(), false); - rulesTO.add(ruleTO); - } - } - - SetStaticNatRulesCommand cmd = new SetStaticNatRulesCommand(rulesTO, router.getVpcId()); - cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); - cmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(guestNetworkId, router.getId())); - cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); - DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); - cmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); - cmds.addCommand(cmd); - } - - @Override - public int getTimeout() { - return -1; - } - - @Override - public boolean isRecurring() { - return false; - } - - @Override - public boolean processAnswers(long agentId, long seq, Answer[] answers) { - return false; - } - - @Override - public boolean processCommands(long agentId, long seq, Command[] commands) { - return false; - } - - @Override - public void processConnect(HostVO host, StartupCommand cmd, boolean forRebalance) throws ConnectionException { - UserContext context = UserContext.current(); - context.setAccountId(1); - List routers = _routerDao.listIsolatedByHostId(host.getId()); - for (DomainRouterVO router : routers) { - if (router.isStopPending()) { - State state = router.getState(); - if (state != State.Stopped && state != State.Destroyed) { - try { - stopRouter(router.getId(), false); - } catch (ResourceUnavailableException e) { - s_logger.warn("Fail to stop router " + router.getInstanceName(), e); - throw new ConnectionException(false, "Fail to stop router " + router.getInstanceName()); - } catch (ConcurrentOperationException e) { - s_logger.warn("Fail to stop router " + router.getInstanceName(), e); - throw new ConnectionException(false, "Fail to stop router " + router.getInstanceName()); - } - } - router.setStopPending(false); - router = _routerDao.persist(router); - } - } - } - - @Override - public AgentControlAnswer processControlCommand(long agentId, AgentControlCommand cmd) { - return null; - } - - @Override - - public boolean processDisconnect(long agentId, Status state) { - return false; - } - - @Override - public boolean processTimeout(long agentId, long seq) { - return false; - } - - protected String getRouterControlIp(long routerId) { - String routerControlIpAddress = null; - List nics = _nicDao.listByVmId(routerId); - for (NicVO n : nics) { - NetworkVO nc = _networkDao.findById(n.getNetworkId()); - if (nc.getTrafficType() == TrafficType.Control) { - routerControlIpAddress = n.getIp4Address(); - } - } - - if(routerControlIpAddress == null) { - s_logger.warn("Unable to find router's control ip in its attached NICs!. routerId: " + routerId); - DomainRouterVO router = _routerDao.findById(routerId); - return router.getPrivateIpAddress(); - } - - return routerControlIpAddress; - } - - - protected String getRouterIpInNetwork(long networkId, long instanceId) { - return _nicDao.getIpAddress(networkId, instanceId); - } - - - @Override - public boolean plugNic(Network network, NicTO nic, VirtualMachineTO vm, ReservationContext context, DeployDestination dest) - throws ConcurrentOperationException, ResourceUnavailableException, - InsufficientCapacityException { - //not supported - throw new UnsupportedOperationException("Plug nic is not supported for vm of type " + vm.getType()); - } - - @Override - public boolean unplugNic(Network network, NicTO nic, VirtualMachineTO vm, ReservationContext context, DeployDestination dest) - throws ConcurrentOperationException, ResourceUnavailableException { - //not supported - throw new UnsupportedOperationException("Unplug nic is not supported for vm of type " + vm.getType()); - } - - @Override - public void prepareStop(VirtualMachineProfile profile){ - //Collect network usage before stopping Vm - VMInstanceVO vm = profile.getVirtualMachine(); - DomainRouterVO router = _routerDao.findById(vm.getId()); - if(router == null){ - return; - } - - String privateIP = router.getPrivateIpAddress(); - - if (privateIP != null) { - List routerNics = _nicDao.listByVmId(router.getId()); - for (Nic routerNic : routerNics) { - Network network = _networkMgr.getNetwork(routerNic.getNetworkId()); - boolean forVpc = router.getVpcId() != null; - if ((forVpc && network.getTrafficType() == TrafficType.Public) || (!forVpc && network.getTrafficType() == TrafficType.Guest)) { - final NetworkUsageCommand usageCmd = new NetworkUsageCommand(privateIP, router.getHostName(), - forVpc, routerNic.getIp4Address()); - UserStatisticsVO previousStats = _statsDao.findBy(router.getAccountId(), - router.getDataCenterIdToDeployIn(), network.getId(), (forVpc ? routerNic.getIp4Address() : null), router.getId(), router.getType().toString()); - NetworkUsageAnswer answer = null; - try { - answer = (NetworkUsageAnswer) _agentMgr.easySend(router.getHostId(), usageCmd); - } catch (Exception e) { - s_logger.warn("Error while collecting network stats from router: "+router.getInstanceName()+" from host: "+router.getHostId(), e); - continue; - } - - if (answer != null) { - if (!answer.getResult()) { - s_logger.warn("Error while collecting network stats from router: "+router.getInstanceName()+" from host: "+router.getHostId() + "; details: " + answer.getDetails()); - continue; - } - Transaction txn = Transaction.open(Transaction.CLOUD_DB); - try { - if ((answer.getBytesReceived() == 0) && (answer.getBytesSent() == 0)) { - s_logger.debug("Recieved and Sent bytes are both 0. Not updating user_statistics"); - continue; - } - txn.start(); - UserStatisticsVO stats = _statsDao.lock(router.getAccountId(), - router.getDataCenterIdToDeployIn(), network.getId(), (forVpc ? routerNic.getIp4Address() : null), router.getId(), router.getType().toString()); - if (stats == null) { - s_logger.warn("unable to find stats for account: " + router.getAccountId()); - continue; - } - - if(previousStats != null - && ((previousStats.getCurrentBytesReceived() != stats.getCurrentBytesReceived()) - || (previousStats.getCurrentBytesSent() != stats.getCurrentBytesSent()))){ - s_logger.debug("Router stats changed from the time NetworkUsageCommand was sent. " + - "Ignoring current answer. Router: "+answer.getRouterName()+" Rcvd: " + - answer.getBytesReceived()+ "Sent: " +answer.getBytesSent()); - continue; - } - - if (stats.getCurrentBytesReceived() > answer.getBytesReceived()) { - if (s_logger.isDebugEnabled()) { - s_logger.debug("Received # of bytes that's less than the last one. " + - "Assuming something went wrong and persisting it. Router: " + - answer.getRouterName()+" Reported: " + answer.getBytesReceived() - + " Stored: " + stats.getCurrentBytesReceived()); - } - stats.setNetBytesReceived(stats.getNetBytesReceived() + stats.getCurrentBytesReceived()); - } - stats.setCurrentBytesReceived(answer.getBytesReceived()); - if (stats.getCurrentBytesSent() > answer.getBytesSent()) { - if (s_logger.isDebugEnabled()) { - s_logger.debug("Received # of bytes that's less than the last one. " + - "Assuming something went wrong and persisting it. Router: " + - answer.getRouterName()+" Reported: " + answer.getBytesSent() - + " Stored: " + stats.getCurrentBytesSent()); - } - stats.setNetBytesSent(stats.getNetBytesSent() + stats.getCurrentBytesSent()); - } - stats.setCurrentBytesSent(answer.getBytesSent()); - _statsDao.update(stats.getId(), stats); - txn.commit(); - } catch (Exception e) { - txn.rollback(); - s_logger.warn("Unable to update user statistics for account: " + router.getAccountId() - + " Rx: " + answer.getBytesReceived() + "; Tx: " + answer.getBytesSent()); - } finally { - txn.close(); - } - } - } - } - } - } - - @Override - public boolean recreateNeeded( - VirtualMachineProfile profile, long hostId, - Commands cmds, ReservationContext context) { - //asssume that if failed to ssh into router, meaning router is crashed - CheckSshAnswer answer = (CheckSshAnswer) cmds.getAnswer("checkSsh"); - if (answer == null || !answer.getResult()) { - return true; - } - - return false; - } -} + return _routerDao.findById(router.getId()); + } else { + return null; + } + } + + @Override + public DomainRouterVO stop(VirtualRouter router, boolean forced, User user, Account caller) throws ConcurrentOperationException, ResourceUnavailableException { + s_logger.debug("Stopping router " + router); + try { + if (_itMgr.advanceStop((DomainRouterVO) router, forced, user, caller)) { + return _routerDao.findById(router.getId()); + } else { + return null; + } + } catch (OperationTimedoutException e) { + throw new CloudRuntimeException("Unable to stop " + router, e); + } + } + + @Override + public boolean applyDhcpEntry(Network network, final NicProfile nic, VirtualMachineProfile profile, + DeployDestination dest, List routers) + throws ResourceUnavailableException { + _userVmDao.loadDetails((UserVmVO) profile.getVirtualMachine()); + + final VirtualMachineProfile updatedProfile = profile; + final boolean isZoneBasic = (dest.getDataCenter().getNetworkType() == NetworkType.Basic); + final Long podId = isZoneBasic ? dest.getPod().getId() : null; + + boolean podLevelException = false; + // for user vm in Basic zone we should try to re-deploy vm in a diff pod if it fails to deploy in original pod; +// so throwing exception with Pod scope + if (isZoneBasic && podId != null && updatedProfile.getVirtualMachine().getType() == VirtualMachine.Type.User + && network.getTrafficType() == TrafficType.Guest && network.getGuestType() == Network.GuestType.Shared) { + podLevelException = true; + } + + return applyRules(network, routers, "dhcp entry", podLevelException, podId, true, new RuleApplier() { + @Override + public boolean execute(Network network, VirtualRouter router) throws ResourceUnavailableException { + // for basic zone, send dhcp/dns information to all routers in the basic network only when +// _dnsBasicZoneUpdates is set to "all" value + Commands cmds = new Commands(OnError.Stop); + if (!(isZoneBasic && router.getPodIdToDeployIn().longValue() != podId.longValue() && _dnsBasicZoneUpdates.equalsIgnoreCase("pod"))) { + NicVO nicVo = _nicDao.findById(nic.getId()); + createDhcpEntryCommand(router, updatedProfile.getVirtualMachine(), nicVo, cmds); + return sendCommandsToRouter(router, cmds); + } + return true; + } + }); + } + + private String findDefaultDnsIp(long userVmId) { + NicVO defaultNic = _nicDao.findDefaultNicForVM(userVmId); + + // check if DNS provider is the domR + if (!_networkMgr.isProviderSupportServiceInNetwork(defaultNic.getNetworkId(), Service.Dns, Provider.VirtualRouter)) { + return null; + } + + NetworkOfferingVO offering = _networkOfferingDao.findById(_networkDao.findById(defaultNic.getNetworkId()).getNetworkOfferingId()); + if (offering.getRedundantRouter()) { + return findGatewayIp(userVmId); + } + + DataCenter dc = _dcDao.findById(_networkMgr.getNetwork(defaultNic.getNetworkId()).getDataCenterId()); + boolean isZoneBasic = (dc.getNetworkType() == NetworkType.Basic); + + // find domR's nic in the network + NicVO domrDefaultNic; + if (isZoneBasic) { + domrDefaultNic = _nicDao.findByNetworkIdTypeAndGateway(defaultNic.getNetworkId(), VirtualMachine.Type.DomainRouter, defaultNic.getGateway()); + } + else { + domrDefaultNic = _nicDao.findByNetworkIdAndType(defaultNic.getNetworkId(), VirtualMachine.Type.DomainRouter); + } + + return domrDefaultNic.getIp4Address(); + } + + private String findGatewayIp(long userVmId) { + NicVO defaultNic = _nicDao.findDefaultNicForVM(userVmId); + return defaultNic.getGateway(); + } + + @Override + public boolean applyUserData(Network network, final NicProfile nic, VirtualMachineProfile profile, DeployDestination dest, List routers) + throws ResourceUnavailableException { + _userVmDao.loadDetails((UserVmVO) profile.getVirtualMachine()); + + final VirtualMachineProfile updatedProfile = profile; + final boolean isZoneBasic = (dest.getDataCenter().getNetworkType() == NetworkType.Basic); + final Long podId = isZoneBasic ? dest.getPod().getId() : null; + + boolean podLevelException = false; + // for user vm in Basic zone we should try to re-deploy vm in a diff pod if it fails to deploy in original pod; +// so throwing exception with Pod scope + if (isZoneBasic && podId != null && updatedProfile.getVirtualMachine().getType() == VirtualMachine.Type.User + && network.getTrafficType() == TrafficType.Guest && network.getGuestType() == Network.GuestType.Shared) { + podLevelException = true; + } + + return applyRules(network, routers, "userdata and password entry", podLevelException, podId, false, new RuleApplier() { + @Override + public boolean execute(Network network, VirtualRouter router) throws ResourceUnavailableException { + // for basic zone, send vm data/password information only to the router in the same pod + Commands cmds = new Commands(OnError.Stop); + if (!(isZoneBasic && router.getPodIdToDeployIn().longValue() != podId.longValue())) { + NicVO nicVo = _nicDao.findById(nic.getId()); + createPasswordCommand(router, updatedProfile, nicVo, cmds); + createVmDataCommand(router, updatedProfile.getVirtualMachine(), nicVo, updatedProfile.getVirtualMachine().getDetail("SSH.PublicKey"), cmds); + return sendCommandsToRouter(router, cmds); + } + return true; + } + }); + } + + @Override + public DomainRouterVO persist(DomainRouterVO router) { + DomainRouterVO virtualRouter = _routerDao.persist(router); + return virtualRouter; + } + + @Override + // FIXME add partial success and STOP state support + public String[] applyVpnUsers(Network network, List users, List routers) throws ResourceUnavailableException { + if (routers == null || routers.isEmpty()) { + s_logger.warn("Failed to add/remove VPN users: no router found for account and zone"); + throw new ResourceUnavailableException("Unable to assign ip addresses, domR doesn't exist for network " + + network.getId(), DataCenter.class, network.getDataCenterId()); + } + + boolean agentResults = true; + + for (DomainRouterVO router : routers) { + if (router.getState() != State.Running) { + s_logger.warn("Failed to add/remove VPN users: router not in running state"); + throw new ResourceUnavailableException("Unable to assign ip addresses, domR is not in right state " + + router.getState(), DataCenter.class, network.getDataCenterId()); + } + + Commands cmds = new Commands(OnError.Continue); + List addUsers = new ArrayList(); + List removeUsers = new ArrayList(); + for (VpnUser user : users) { + if (user.getState() == VpnUser.State.Add || user.getState() == VpnUser.State.Active) { + addUsers.add(user); + } else if (user.getState() == VpnUser.State.Revoke) { + removeUsers.add(user); + } + } + + VpnUsersCfgCommand cmd = new VpnUsersCfgCommand(addUsers, removeUsers); + cmd.setAccessDetail(NetworkElementCommand.ACCOUNT_ID, String.valueOf(router.getAccountId())); + cmd.setAccessDetail(NetworkElementCommand.GUEST_NETWORK_CIDR, network.getCidr()); + cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); + cmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(network.getId(), router.getId())); + cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); + DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); + cmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); + + cmds.addCommand(cmd); + + // Currently we receive just one answer from the agent. In the future we have to parse individual answers +// and set + // results accordingly + boolean agentResult = sendCommandsToRouter(router, cmds); + agentResults = agentResults && agentResult; + } + + String[] result = new String[users.size()]; + for (int i = 0; i < result.length; i++) { + if (agentResults) { + result[i] = null; + } else { + result[i] = String.valueOf(agentResults); + } + } + + return result; + } + + @Override + public DomainRouterVO findById(long id) { + return _routerDao.findById(id); + } + + @Override + public DomainRouterVO findByName(String name) { + if (!VirtualMachineName.isValidRouterName(name)) { + return null; + } + + return _routerDao.findById(VirtualMachineName.getRouterId(name)); + } + + @Override + @ActionEvent(eventType = EventTypes.EVENT_ROUTER_START, eventDescription = "starting router Vm", async = true) + public VirtualRouter startRouter(long id) throws ResourceUnavailableException, InsufficientCapacityException, ConcurrentOperationException { + return startRouter(id, true); + } + + @Override + public VirtualRouter startRouter(long routerId, boolean reprogramNetwork) throws ResourceUnavailableException, + InsufficientCapacityException, ConcurrentOperationException { + Account caller = UserContext.current().getCaller(); + User callerUser = _accountMgr.getActiveUser(UserContext.current().getCallerUserId()); + + // verify parameters + DomainRouterVO router = _routerDao.findById(routerId); + if (router == null) { + throw new InvalidParameterValueException("Unable to find router by id", null); + } + _accountMgr.checkAccess(caller, null, true, router); + + Account owner = _accountMgr.getAccount(router.getAccountId()); + + // Check if all networks are implemented for the domR; if not - implement them + DataCenter dc = _dcDao.findById(router.getDataCenterIdToDeployIn()); + HostPodVO pod = null; + if (router.getPodIdToDeployIn() != null) { + pod = _podDao.findById(router.getPodIdToDeployIn()); + } + DeployDestination dest = new DeployDestination(dc, pod, null, null); + + ReservationContext context = new ReservationContextImpl(null, null, callerUser, owner); + + List nics = _nicDao.listByVmId(routerId); + + for (NicVO nic : nics) { + if (!_networkMgr.startNetwork(nic.getNetworkId(), dest, context)) { + s_logger.warn("Failed to start network id=" + nic.getNetworkId() + " as a part of domR start"); + throw new CloudRuntimeException("Failed to start network id=" + nic.getNetworkId() + " as a part of domR start"); + } + } + + UserVO user = _userDao.findById(UserContext.current().getCallerUserId()); + Map params = new HashMap(); + if (reprogramNetwork) { + params.put(Param.ReProgramGuestNetworks, true); + } else { + params.put(Param.ReProgramGuestNetworks, false); + } + VirtualRouter virtualRouter = startVirtualRouter(router, user, caller, params); + if (virtualRouter == null) { + throw new CloudRuntimeException("Failed to start router with id " + routerId); + } + return virtualRouter; + } + + private void createAssociateIPCommands(final VirtualRouter router, final List ips, Commands cmds, long vmId) { + + // Ensure that in multiple vlans case we first send all ip addresses of vlan1, then all ip addresses of vlan2, +// etc.. + Map> vlanIpMap = new HashMap>(); + for (final PublicIpAddress ipAddress : ips) { + String vlanTag = ipAddress.getVlanTag(); + ArrayList ipList = vlanIpMap.get(vlanTag); + if (ipList == null) { + ipList = new ArrayList(); + } + // domR doesn't support release for sourceNat IP address; so reset the state + if (ipAddress.isSourceNat() && ipAddress.getState() == IpAddress.State.Releasing) { + ipAddress.setState(IpAddress.State.Allocated); + } + ipList.add(ipAddress); + vlanIpMap.put(vlanTag, ipList); + } + + for (Map.Entry> vlanAndIp : vlanIpMap.entrySet()) { + List ipAddrList = vlanAndIp.getValue(); + // Source nat ip address should always be sent first + Collections.sort(ipAddrList, new Comparator() { + @Override + public int compare(PublicIpAddress o1, PublicIpAddress o2) { + boolean s1 = o1.isSourceNat(); + boolean s2 = o2.isSourceNat(); + return (s1 ^ s2) ? ((s1 ^ true) ? 1 : -1) : 0; + } + }); + + // Get network rate - required for IpAssoc + Integer networkRate = _networkMgr.getNetworkRate(ipAddrList.get(0).getNetworkId(), router.getId()); + Network network = _networkMgr.getNetwork(ipAddrList.get(0).getNetworkId()); + + IpAddressTO[] ipsToSend = new IpAddressTO[ipAddrList.size()]; + int i = 0; + boolean firstIP = true; + + for (final PublicIpAddress ipAddr : ipAddrList) { + + boolean add = (ipAddr.getState() == IpAddress.State.Releasing ? false : true); + boolean sourceNat = ipAddr.isSourceNat(); + /* enable sourceNAT for the first ip of the public interface */ + if (firstIP) { + sourceNat = true; + } + String vlanId = ipAddr.getVlanTag(); + String vlanGateway = ipAddr.getGateway(); + String vlanNetmask = ipAddr.getNetmask(); + String vifMacAddress = ipAddr.getMacAddress(); + + String vmGuestAddress = null; + + IpAddressTO ip = new IpAddressTO(ipAddr.getAccountId(), ipAddr.getAddress().addr(), add, firstIP, + sourceNat, vlanId, vlanGateway, vlanNetmask, vifMacAddress, vmGuestAddress, networkRate, ipAddr.isOneToOneNat()); + + ip.setTrafficType(network.getTrafficType()); + ip.setNetworkName(_networkMgr.getNetworkTag(router.getHypervisorType(), network)); + ipsToSend[i++] = ip; + /* send the firstIP = true for the first Add, this is to create primary on interface */ + if (!firstIP || add) { + firstIP = false; + } + } + IpAssocCommand cmd = new IpAssocCommand(ipsToSend); + cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); + cmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(ipAddrList.get(0).getNetworkId(), router.getId())); + cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); + DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); + cmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); + + cmds.addCommand("IPAssocCommand", cmd); + } + } + + private void createApplyPortForwardingRulesCommands(List rules, VirtualRouter router, Commands cmds, long guestNetworkId) { + List rulesTO = null; + if (rules != null) { + rulesTO = new ArrayList(); + for (PortForwardingRule rule : rules) { + IpAddress sourceIp = _networkMgr.getIp(rule.getSourceIpAddressId()); + PortForwardingRuleTO ruleTO = new PortForwardingRuleTO(rule, null, sourceIp.getAddress().addr()); + rulesTO.add(ruleTO); + } + } + + SetPortForwardingRulesCommand cmd = null; + + if (router.getVpcId() != null) { + cmd = new SetPortForwardingRulesVpcCommand(rulesTO); + } else { + cmd = new SetPortForwardingRulesCommand(rulesTO); + } + + cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); + cmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(guestNetworkId, router.getId())); + cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); + DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); + cmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); + + cmds.addCommand(cmd); + } + + private void createApplyStaticNatRulesCommands(List rules, VirtualRouter router, Commands cmds, long guestNetworkId) { + List rulesTO = null; + if (rules != null) { + rulesTO = new ArrayList(); + for (StaticNatRule rule : rules) { + IpAddress sourceIp = _networkMgr.getIp(rule.getSourceIpAddressId()); + StaticNatRuleTO ruleTO = new StaticNatRuleTO(rule, null, sourceIp.getAddress().addr(), rule.getDestIpAddress()); + rulesTO.add(ruleTO); + } + } + + SetStaticNatRulesCommand cmd = new SetStaticNatRulesCommand(rulesTO, router.getVpcId()); + cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); + cmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(guestNetworkId, router.getId())); + cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); + DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); + cmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); + cmds.addCommand(cmd); + } + + private void createApplyLoadBalancingRulesCommands(List rules, VirtualRouter router, Commands cmds, long guestNetworkId) { + + LoadBalancerTO[] lbs = new LoadBalancerTO[rules.size()]; + int i = 0; + for (LoadBalancingRule rule : rules) { + boolean revoked = (rule.getState().equals(FirewallRule.State.Revoke)); + String protocol = rule.getProtocol(); + String algorithm = rule.getAlgorithm(); + String uuid = rule.getUuid(); + + String srcIp = _networkMgr.getIp(rule.getSourceIpAddressId()).getAddress().addr(); + int srcPort = rule.getSourcePortStart(); + List destinations = rule.getDestinations(); + List stickinessPolicies = rule.getStickinessPolicies(); + LoadBalancerTO lb = new LoadBalancerTO(uuid, srcIp, srcPort, protocol, algorithm, revoked, false, destinations, stickinessPolicies); + lbs[i++] = lb; + } + String routerPublicIp = null; + + if (router instanceof DomainRouterVO) { + DomainRouterVO domr = _routerDao.findById(router.getId()); + routerPublicIp = domr.getPublicIpAddress(); + } + + Network guestNetwork = _networkMgr.getNetwork(guestNetworkId); + Nic nic = _nicDao.findByInstanceIdAndNetworkId(guestNetwork.getId(), router.getId()); + NicProfile nicProfile = new NicProfile(nic, guestNetwork, nic.getBroadcastUri(), nic.getIsolationUri(), + _networkMgr.getNetworkRate(guestNetwork.getId(), router.getId()), + _networkMgr.isSecurityGroupSupportedInNetwork(guestNetwork), + _networkMgr.getNetworkTag(router.getHypervisorType(), guestNetwork)); + + LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(lbs, routerPublicIp, + getRouterIpInNetwork(guestNetworkId, router.getId()), router.getPrivateIpAddress(), + _itMgr.toNicTO(nicProfile, router.getHypervisorType()), router.getVpcId()); + + cmd.lbStatsVisibility = _configDao.getValue(Config.NetworkLBHaproxyStatsVisbility.key()); + cmd.lbStatsUri = _configDao.getValue(Config.NetworkLBHaproxyStatsUri.key()); + cmd.lbStatsAuth = _configDao.getValue(Config.NetworkLBHaproxyStatsAuth.key()); + cmd.lbStatsPort = _configDao.getValue(Config.NetworkLBHaproxyStatsPort.key()); + + cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); + cmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(guestNetworkId, router.getId())); + cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); + DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); + cmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); + cmds.addCommand(cmd); + + } + + private void createApplyVpnCommands(RemoteAccessVpn vpn, VirtualRouter router, Commands cmds) { + List vpnUsers = _vpnUsersDao.listByAccount(vpn.getAccountId()); + List addUsers = new ArrayList(); + List removeUsers = new ArrayList(); + for (VpnUser user : vpnUsers) { + if (user.getState() == VpnUser.State.Add) { + addUsers.add(user); + } else if (user.getState() == VpnUser.State.Revoke) { + removeUsers.add(user); + } + } + + VpnUsersCfgCommand addUsersCmd = new VpnUsersCfgCommand(addUsers, removeUsers); + addUsersCmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); + addUsersCmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(vpn.getNetworkId(), router.getId())); + addUsersCmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); + + IpAddress ip = _networkMgr.getIp(vpn.getServerAddressId()); + + RemoteAccessVpnCfgCommand startVpnCmd = new RemoteAccessVpnCfgCommand(true, ip.getAddress().addr(), + vpn.getLocalIp(), vpn.getIpRange(), vpn.getIpsecPresharedKey()); + startVpnCmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); + startVpnCmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(vpn.getNetworkId(), router.getId())); + startVpnCmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); + DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); + startVpnCmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); + + cmds.addCommand("users", addUsersCmd); + cmds.addCommand("startVpn", startVpnCmd); + } + + private void createPasswordCommand(VirtualRouter router, VirtualMachineProfile profile, NicVO nic, Commands cmds) { + String password = (String) profile.getParameter(VirtualMachineProfile.Param.VmPassword); + DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); + + // password should be set only on default network element + if (password != null && nic.isDefaultNic()) { + final String encodedPassword = PasswordGenerator.rot13(password); + SavePasswordCommand cmd = new SavePasswordCommand(encodedPassword, nic.getIp4Address(), profile.getVirtualMachine().getHostName()); + cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); + cmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(nic.getNetworkId(), router.getId())); + cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); + cmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); + + cmds.addCommand("password", cmd); + } + + } + + private void createVmDataCommand(VirtualRouter router, UserVm vm, NicVO nic, String publicKey, Commands cmds) { + String serviceOffering = _serviceOfferingDao.findByIdIncludingRemoved(vm.getServiceOfferingId()).getDisplayText(); + String zoneName = _dcDao.findById(router.getDataCenterIdToDeployIn()).getName(); + cmds.addCommand("vmdata", + generateVmDataCommand(router, nic.getIp4Address(), vm.getUserData(), serviceOffering, zoneName, nic.getIp4Address(), + vm.getHostName(), vm.getInstanceName(), vm.getId(), publicKey, nic.getNetworkId())); + + } + + private void createVmDataCommandForVMs(DomainRouterVO router, Commands cmds, long guestNetworkId) { + List vms = _userVmDao.listByNetworkIdAndStates(guestNetworkId, State.Running, State.Migrating, State.Stopping); + DataCenterVO dc = _dcDao.findById(router.getDataCenterIdToDeployIn()); + for (UserVmVO vm : vms) { + boolean createVmData = true; + if (dc.getNetworkType() == NetworkType.Basic && router.getPodIdToDeployIn().longValue() != vm.getPodIdToDeployIn().longValue()) { + createVmData = false; + } + + if (createVmData) { + NicVO nic = _nicDao.findByInstanceIdAndNetworkId(guestNetworkId, vm.getId()); + if (nic != null) { + s_logger.debug("Creating user data entry for vm " + vm + " on domR " + router); + createVmDataCommand(router, vm, nic, null, cmds); + } + } + } + } + + private void createDhcpEntryCommand(VirtualRouter router, UserVm vm, NicVO nic, Commands cmds) { + DhcpEntryCommand dhcpCommand = new DhcpEntryCommand(nic.getMacAddress(), nic.getIp4Address(), vm.getHostName()); + DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); + String gatewayIp = findGatewayIp(vm.getId()); + if (!gatewayIp.equals(nic.getGateway())) { + GuestOSVO guestOS = _guestOSDao.findById(vm.getGuestOSId()); + // Don't set dhcp:router option for non-default nic on CentOS/RHEL, because they would set routing on wrong +// interface + // This is tricky, we may need to update this when we have more information on various OS's behavior + if (guestOS.getDisplayName().startsWith("CentOS") || guestOS.getDisplayName().startsWith("Red Hat Enterprise")) { + gatewayIp = "0.0.0.0"; + } + } + dhcpCommand.setDefaultRouter(gatewayIp); + dhcpCommand.setDefaultDns(findDefaultDnsIp(vm.getId())); + + dhcpCommand.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); + dhcpCommand.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); + dhcpCommand.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(nic.getNetworkId(), router.getId())); + dhcpCommand.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); + + cmds.addCommand("dhcp", dhcpCommand); + } + + private void createDhcpEntryCommandsForVMs(DomainRouterVO router, Commands cmds, long guestNetworkId) { + List vms = _userVmDao.listByNetworkIdAndStates(guestNetworkId, State.Running, State.Migrating, State.Stopping); + DataCenterVO dc = _dcDao.findById(router.getDataCenterIdToDeployIn()); + for (UserVmVO vm : vms) { + boolean createDhcp = true; + if (dc.getNetworkType() == NetworkType.Basic && router.getPodIdToDeployIn().longValue() != vm.getPodIdToDeployIn().longValue() + && _dnsBasicZoneUpdates.equalsIgnoreCase("pod")) { + createDhcp = false; + } + if (createDhcp) { + NicVO nic = _nicDao.findByInstanceIdAndNetworkId(guestNetworkId, vm.getId()); + if (nic != null) { + s_logger.debug("Creating dhcp entry for vm " + vm + " on domR " + router + "."); + createDhcpEntryCommand(router, vm, nic, cmds); + } + } + } + } + + protected boolean sendCommandsToRouter(final VirtualRouter router, Commands cmds) throws AgentUnavailableException { + Answer[] answers = null; + try { + answers = _agentMgr.send(router.getHostId(), cmds); + } catch (OperationTimedoutException e) { + s_logger.warn("Timed Out", e); + throw new AgentUnavailableException("Unable to send commands to virtual router ", router.getHostId(), e); + } + + if (answers == null) { + return false; + } + + if (answers.length != cmds.size()) { + return false; + } + + // FIXME: Have to return state for individual command in the future + boolean result = true; + if (answers.length > 0) { + for (Answer answer : answers) { + if (!answer.getResult()) { + result = false; + break; + } + } + } + return result; + } + + protected void handleSingleWorkingRedundantRouter(List connectedRouters, List disconnectedRouters, String reason) throws ResourceUnavailableException + { + if (connectedRouters.isEmpty() || disconnectedRouters.isEmpty()) { + return; + } + if (connectedRouters.size() != 1 || disconnectedRouters.size() != 1) { + s_logger.warn("How many redundant routers do we have?? "); + return; + } + if (!connectedRouters.get(0).getIsRedundantRouter()) { + throw new ResourceUnavailableException("Who is calling this with non-redundant router or non-domain router?", + DataCenter.class, connectedRouters.get(0).getDataCenterIdToDeployIn()); + } + if (!disconnectedRouters.get(0).getIsRedundantRouter()) { + throw new ResourceUnavailableException("Who is calling this with non-redundant router or non-domain router?", + DataCenter.class, disconnectedRouters.get(0).getDataCenterIdToDeployIn()); + } + + DomainRouterVO connectedRouter = (DomainRouterVO) connectedRouters.get(0); + DomainRouterVO disconnectedRouter = (DomainRouterVO) disconnectedRouters.get(0); + + if (s_logger.isDebugEnabled()) { + s_logger.debug("About to stop the router " + disconnectedRouter.getInstanceName() + " due to: " + reason); + } + String title = "Virtual router " + disconnectedRouter.getInstanceName() + " would be stopped after connecting back, due to " + reason; + String context = "Virtual router (name: " + disconnectedRouter.getInstanceName() + ", id: " + disconnectedRouter.getId() + ") would be stopped after connecting back, due to: " + reason; + _alertMgr.sendAlert(AlertManager.ALERT_TYPE_DOMAIN_ROUTER, + disconnectedRouter.getDataCenterIdToDeployIn(), disconnectedRouter.getPodIdToDeployIn(), title, context); + disconnectedRouter.setStopPending(true); + disconnectedRouter = _routerDao.persist(disconnectedRouter); + + int connRouterPR = getRealPriority(connectedRouter); + int disconnRouterPR = getRealPriority(disconnectedRouter); + if (connRouterPR < disconnRouterPR) { + // connRouterPR < disconnRouterPR, they won't equal at anytime + if (!connectedRouter.getIsPriorityBumpUp()) { + final BumpUpPriorityCommand command = new BumpUpPriorityCommand(); + command.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(connectedRouter.getId())); + command.setAccessDetail(NetworkElementCommand.ROUTER_NAME, connectedRouter.getInstanceName()); + final Answer answer = _agentMgr.easySend(connectedRouter.getHostId(), command); + if (!answer.getResult()) { + s_logger.error("Failed to bump up " + connectedRouter.getInstanceName() + "'s priority! " + answer.getDetails()); + } + } else { + String t = "Can't bump up virtual router " + connectedRouter.getInstanceName() + "'s priority due to it's already bumped up!"; + _alertMgr.sendAlert(AlertManager.ALERT_TYPE_DOMAIN_ROUTER, + connectedRouter.getDataCenterIdToDeployIn(), connectedRouter.getPodIdToDeployIn(), t, t); + } + } + } + + @Override + public boolean associatePublicIP(Network network, final List ipAddress, List routers) + throws ResourceUnavailableException { + if (ipAddress == null || ipAddress.isEmpty()) { + s_logger.debug("No ip association rules to be applied for network " + network.getId()); + return true; + } + return applyRules(network, routers, "ip association", false, null, false, new RuleApplier() { + @Override + public boolean execute(Network network, VirtualRouter router) throws ResourceUnavailableException { + Commands cmds = new Commands(OnError.Continue); + createAssociateIPCommands(router, ipAddress, cmds, 0); + return sendCommandsToRouter(router, cmds); + } + }); + } + + @Override + public boolean applyFirewallRules(Network network, final List rules, List routers) throws ResourceUnavailableException { + if (rules == null || rules.isEmpty()) { + s_logger.debug("No firewall rules to be applied for network " + network.getId()); + return true; + } + return applyRules(network, routers, "firewall rules", false, null, false, new RuleApplier() { + @Override + public boolean execute(Network network, VirtualRouter router) throws ResourceUnavailableException { + if (rules.get(0).getPurpose() == Purpose.LoadBalancing) { + // for load balancer we have to resend all lb rules for the network + List lbs = _loadBalancerDao.listByNetworkId(network.getId()); + List lbRules = new ArrayList(); + for (LoadBalancerVO lb : lbs) { + List dstList = _lbMgr.getExistingDestinations(lb.getId()); + List policyList = _lbMgr.getStickinessPolicies(lb.getId()); + LoadBalancingRule loadBalancing = new LoadBalancingRule(lb, dstList, policyList); + lbRules.add(loadBalancing); + } + return sendLBRules(router, lbRules, network.getId()); + } else if (rules.get(0).getPurpose() == Purpose.PortForwarding) { + return sendPortForwardingRules(router, (List) rules, network.getId()); + } else if (rules.get(0).getPurpose() == Purpose.StaticNat) { + return sendStaticNatRules(router, (List) rules, network.getId()); + } else if (rules.get(0).getPurpose() == Purpose.Firewall) { + return sendFirewallRules(router, (List) rules, network.getId()); + } else { + s_logger.warn("Unable to apply rules of purpose: " + rules.get(0).getPurpose()); + return false; + } + } + }); + } + + protected boolean sendLBRules(VirtualRouter router, List rules, long guestNetworkId) throws ResourceUnavailableException { + Commands cmds = new Commands(OnError.Continue); + createApplyLoadBalancingRulesCommands(rules, router, cmds, guestNetworkId); + return sendCommandsToRouter(router, cmds); + } + + protected boolean sendPortForwardingRules(VirtualRouter router, List rules, long guestNetworkId) throws ResourceUnavailableException { + Commands cmds = new Commands(OnError.Continue); + createApplyPortForwardingRulesCommands(rules, router, cmds, guestNetworkId); + return sendCommandsToRouter(router, cmds); + } + + protected boolean sendStaticNatRules(VirtualRouter router, List rules, long guestNetworkId) throws ResourceUnavailableException { + Commands cmds = new Commands(OnError.Continue); + createApplyStaticNatRulesCommands(rules, router, cmds, guestNetworkId); + return sendCommandsToRouter(router, cmds); + } + + @Override + public List getRoutersForNetwork(long networkId) { + List routers = _routerDao.findByNetwork(networkId); + List vrs = new ArrayList(routers.size()); + for (DomainRouterVO router : routers) { + vrs.add(router); + } + return vrs; + } + + private void createFirewallRulesCommands(List rules, VirtualRouter router, Commands cmds, long guestNetworkId) { + List rulesTO = null; + if (rules != null) { + rulesTO = new ArrayList(); + for (FirewallRule rule : rules) { + IpAddress sourceIp = _networkMgr.getIp(rule.getSourceIpAddressId()); + FirewallRuleTO ruleTO = new FirewallRuleTO(rule, null, sourceIp.getAddress().addr()); + rulesTO.add(ruleTO); + } + } + + SetFirewallRulesCommand cmd = new SetFirewallRulesCommand(rulesTO); + cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); + cmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(guestNetworkId, router.getId())); + cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); + DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); + cmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); + cmds.addCommand(cmd); + } + + protected boolean sendFirewallRules(VirtualRouter router, List rules, long guestNetworkId) throws ResourceUnavailableException { + Commands cmds = new Commands(OnError.Continue); + createFirewallRulesCommands(rules, router, cmds, guestNetworkId); + return sendCommandsToRouter(router, cmds); + } + + @Override + public String getDnsBasicZoneUpdate() { + return _dnsBasicZoneUpdates; + } + + protected interface RuleApplier { + boolean execute(Network network, VirtualRouter router) throws ResourceUnavailableException; + } + + protected boolean applyRules(Network network, List routers, String typeString, + boolean isPodLevelException, Long podId, boolean failWhenDisconnect, RuleApplier applier) throws ResourceUnavailableException { + if (routers == null || routers.isEmpty()) { + s_logger.warn("Unable to apply " + typeString + ", virtual router doesn't exist in the network " + network.getId()); + throw new ResourceUnavailableException("Unable to apply " + typeString, DataCenter.class, network.getDataCenterId()); + } + + DataCenter dc = _dcDao.findById(network.getDataCenterId()); + boolean isZoneBasic = (dc.getNetworkType() == NetworkType.Basic); + + // isPodLevelException and podId is only used for basic zone + assert !((!isZoneBasic && isPodLevelException) || (isZoneBasic && isPodLevelException && podId == null)); + + List connectedRouters = new ArrayList(); + List disconnectedRouters = new ArrayList(); + boolean result = true; + String msg = "Unable to apply " + typeString + " on disconnected router "; + for (VirtualRouter router : routers) { + if (router.getState() == State.Running) { + s_logger.debug("Applying " + typeString + " in network " + network); + + if (router.isStopPending()) { + if (_hostDao.findById(router.getHostId()).getStatus() == Status.Up) { + throw new ResourceUnavailableException("Unable to process due to the stop pending router " + + router.getInstanceName() + " haven't been stopped after it's host coming back!", + DataCenter.class, router.getDataCenterIdToDeployIn()); + } + s_logger.debug("Router " + router.getInstanceName() + " is stop pending, so not sending apply " + + typeString + " commands to the backend"); + continue; + } + try { + result = applier.execute(network, router); + connectedRouters.add(router); + } catch (AgentUnavailableException e) { + s_logger.warn(msg + router.getInstanceName(), e); + disconnectedRouters.add(router); + } + + // If rules fail to apply on one domR and not due to disconnection, no need to proceed with the rest + if (!result) { + if (isZoneBasic && isPodLevelException) { + throw new ResourceUnavailableException("Unable to apply " + typeString + " on router ", Pod.class, podId); + } + throw new ResourceUnavailableException("Unable to apply " + typeString + " on router ", DataCenter.class, + router.getDataCenterIdToDeployIn()); + } + + } else if (router.getState() == State.Stopped || router.getState() == State.Stopping) { + s_logger.debug("Router " + router.getInstanceName() + " is in " + router.getState() + + ", so not sending apply " + typeString + " commands to the backend"); + } else { + s_logger.warn("Unable to apply " + typeString + ", virtual router is not in the right state " + router.getState()); + if (isZoneBasic && isPodLevelException) { + throw new ResourceUnavailableException("Unable to apply " + typeString + + ", virtual router is not in the right state", Pod.class, podId); + } + throw new ResourceUnavailableException("Unable to apply " + typeString + + ", virtual router is not in the right state", DataCenter.class, router.getDataCenterIdToDeployIn()); + } + } + + if (!connectedRouters.isEmpty()) { + if (!isZoneBasic && !disconnectedRouters.isEmpty() && disconnectedRouters.get(0).getIsRedundantRouter()) { + // These disconnected redundant virtual routers are out of sync now, stop them for synchronization + handleSingleWorkingRedundantRouter(connectedRouters, disconnectedRouters, msg); + } + } else if (!disconnectedRouters.isEmpty()) { + for (VirtualRouter router : disconnectedRouters) { + if (s_logger.isDebugEnabled()) { + s_logger.debug(msg + router.getInstanceName() + "(" + router.getId() + ")"); + } + } + if (isZoneBasic && isPodLevelException) { + throw new ResourceUnavailableException(msg, Pod.class, podId); + } + throw new ResourceUnavailableException(msg, DataCenter.class, disconnectedRouters.get(0).getDataCenterIdToDeployIn()); + } + + result = true; + if (failWhenDisconnect) { + result = !connectedRouters.isEmpty(); + } + return result; + } + + @Override + public boolean applyStaticNats(Network network, final List rules, List routers) throws ResourceUnavailableException { + if (rules == null || rules.isEmpty()) { + s_logger.debug("No static nat rules to be applied for network " + network.getId()); + return true; + } + return applyRules(network, routers, "static nat rules", false, null, false, new RuleApplier() { + @Override + public boolean execute(Network network, VirtualRouter router) throws ResourceUnavailableException { + return applyStaticNat(router, rules, network.getId()); + } + }); + } + + protected boolean applyStaticNat(VirtualRouter router, List rules, long guestNetworkId) throws ResourceUnavailableException { + Commands cmds = new Commands(OnError.Continue); + createApplyStaticNatCommands(rules, router, cmds, guestNetworkId); + return sendCommandsToRouter(router, cmds); + } + + private void createApplyStaticNatCommands(List rules, VirtualRouter router, Commands cmds, + long guestNetworkId) { + List rulesTO = null; + if (rules != null) { + rulesTO = new ArrayList(); + for (StaticNat rule : rules) { + IpAddress sourceIp = _networkMgr.getIp(rule.getSourceIpAddressId()); + StaticNatRuleTO ruleTO = new StaticNatRuleTO(0, sourceIp.getAddress().addr(), null, + null, rule.getDestIpAddress(), null, null, null, rule.isForRevoke(), false); + rulesTO.add(ruleTO); + } + } + + SetStaticNatRulesCommand cmd = new SetStaticNatRulesCommand(rulesTO, router.getVpcId()); + cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); + cmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(guestNetworkId, router.getId())); + cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); + DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); + cmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); + cmds.addCommand(cmd); + } + + @Override + public int getTimeout() { + return -1; + } + + @Override + public boolean isRecurring() { + return false; + } + + @Override + public boolean processAnswers(long agentId, long seq, Answer[] answers) { + return false; + } + + @Override + public boolean processCommands(long agentId, long seq, Command[] commands) { + return false; + } + + @Override + public void processConnect(HostVO host, StartupCommand cmd, boolean forRebalance) throws ConnectionException { + UserContext context = UserContext.current(); + context.setAccountId(1); + List routers = _routerDao.listIsolatedByHostId(host.getId()); + for (DomainRouterVO router : routers) { + if (router.isStopPending()) { + State state = router.getState(); + if (state != State.Stopped && state != State.Destroyed) { + try { + stopRouter(router.getId(), false); + } catch (ResourceUnavailableException e) { + s_logger.warn("Fail to stop router " + router.getInstanceName(), e); + throw new ConnectionException(false, "Fail to stop router " + router.getInstanceName()); + } catch (ConcurrentOperationException e) { + s_logger.warn("Fail to stop router " + router.getInstanceName(), e); + throw new ConnectionException(false, "Fail to stop router " + router.getInstanceName()); + } + } + router.setStopPending(false); + router = _routerDao.persist(router); + } + } + } + + @Override + public AgentControlAnswer processControlCommand(long agentId, AgentControlCommand cmd) { + return null; + } + + @Override + public boolean processDisconnect(long agentId, Status state) { + return false; + } + + @Override + public boolean processTimeout(long agentId, long seq) { + return false; + } + + protected String getRouterControlIp(long routerId) { + String routerControlIpAddress = null; + List nics = _nicDao.listByVmId(routerId); + for (NicVO n : nics) { + NetworkVO nc = _networkDao.findById(n.getNetworkId()); + if (nc.getTrafficType() == TrafficType.Control) { + routerControlIpAddress = n.getIp4Address(); + } + } + + if (routerControlIpAddress == null) { + s_logger.warn("Unable to find router's control ip in its attached NICs!. routerId: " + routerId); + DomainRouterVO router = _routerDao.findById(routerId); + return router.getPrivateIpAddress(); + } + + return routerControlIpAddress; + } + + protected String getRouterIpInNetwork(long networkId, long instanceId) { + return _nicDao.getIpAddress(networkId, instanceId); + } + + @Override + public boolean plugNic(Network network, NicTO nic, VirtualMachineTO vm, ReservationContext context, DeployDestination dest) + throws ConcurrentOperationException, ResourceUnavailableException, + InsufficientCapacityException { + // not supported + throw new UnsupportedOperationException("Plug nic is not supported for vm of type " + vm.getType()); + } + + @Override + public boolean unplugNic(Network network, NicTO nic, VirtualMachineTO vm, ReservationContext context, DeployDestination dest) + throws ConcurrentOperationException, ResourceUnavailableException { + // not supported + throw new UnsupportedOperationException("Unplug nic is not supported for vm of type " + vm.getType()); + } + + @Override + public void prepareStop(VirtualMachineProfile profile) { + // Collect network usage before stopping Vm + VMInstanceVO vm = profile.getVirtualMachine(); + DomainRouterVO router = _routerDao.findById(vm.getId()); + if (router == null) { + return; + } + + String privateIP = router.getPrivateIpAddress(); + + if (privateIP != null) { + List routerNics = _nicDao.listByVmId(router.getId()); + for (Nic routerNic : routerNics) { + Network network = _networkMgr.getNetwork(routerNic.getNetworkId()); + boolean forVpc = router.getVpcId() != null; + if ((forVpc && network.getTrafficType() == TrafficType.Public) || (!forVpc && network.getTrafficType() == TrafficType.Guest)) { + final NetworkUsageCommand usageCmd = new NetworkUsageCommand(privateIP, router.getHostName(), + forVpc, routerNic.getIp4Address()); + UserStatisticsVO previousStats = _statsDao.findBy(router.getAccountId(), + router.getDataCenterIdToDeployIn(), network.getId(), (forVpc ? routerNic.getIp4Address() : null), router.getId(), router.getType().toString()); + NetworkUsageAnswer answer = null; + try { + answer = (NetworkUsageAnswer) _agentMgr.easySend(router.getHostId(), usageCmd); + } catch (Exception e) { + s_logger.warn("Error while collecting network stats from router: " + router.getInstanceName() + " from host: " + router.getHostId(), e); + continue; + } + + if (answer != null) { + if (!answer.getResult()) { + s_logger.warn("Error while collecting network stats from router: " + router.getInstanceName() + " from host: " + router.getHostId() + "; details: " + answer.getDetails()); + continue; + } + Transaction txn = Transaction.open(Transaction.CLOUD_DB); + try { + if ((answer.getBytesReceived() == 0) && (answer.getBytesSent() == 0)) { + s_logger.debug("Recieved and Sent bytes are both 0. Not updating user_statistics"); + continue; + } + txn.start(); + UserStatisticsVO stats = _statsDao.lock(router.getAccountId(), + router.getDataCenterIdToDeployIn(), network.getId(), (forVpc ? routerNic.getIp4Address() : null), router.getId(), router.getType().toString()); + if (stats == null) { + s_logger.warn("unable to find stats for account: " + router.getAccountId()); + continue; + } + + if (previousStats != null + && ((previousStats.getCurrentBytesReceived() != stats.getCurrentBytesReceived()) + || (previousStats.getCurrentBytesSent() != stats.getCurrentBytesSent()))) { + s_logger.debug("Router stats changed from the time NetworkUsageCommand was sent. " + + "Ignoring current answer. Router: " + answer.getRouterName() + " Rcvd: " + + answer.getBytesReceived() + "Sent: " + answer.getBytesSent()); + continue; + } + + if (stats.getCurrentBytesReceived() > answer.getBytesReceived()) { + if (s_logger.isDebugEnabled()) { + s_logger.debug("Received # of bytes that's less than the last one. " + + "Assuming something went wrong and persisting it. Router: " + + answer.getRouterName() + " Reported: " + answer.getBytesReceived() + + " Stored: " + stats.getCurrentBytesReceived()); + } + stats.setNetBytesReceived(stats.getNetBytesReceived() + stats.getCurrentBytesReceived()); + } + stats.setCurrentBytesReceived(answer.getBytesReceived()); + if (stats.getCurrentBytesSent() > answer.getBytesSent()) { + if (s_logger.isDebugEnabled()) { + s_logger.debug("Received # of bytes that's less than the last one. " + + "Assuming something went wrong and persisting it. Router: " + + answer.getRouterName() + " Reported: " + answer.getBytesSent() + + " Stored: " + stats.getCurrentBytesSent()); + } + stats.setNetBytesSent(stats.getNetBytesSent() + stats.getCurrentBytesSent()); + } + stats.setCurrentBytesSent(answer.getBytesSent()); + _statsDao.update(stats.getId(), stats); + txn.commit(); + } catch (Exception e) { + txn.rollback(); + s_logger.warn("Unable to update user statistics for account: " + router.getAccountId() + + " Rx: " + answer.getBytesReceived() + "; Tx: " + answer.getBytesSent()); + } finally { + txn.close(); + } + } + } + } + } + } + + @Override + public boolean recreateNeeded( + VirtualMachineProfile profile, long hostId, + Commands cmds, ReservationContext context) { + // asssume that if failed to ssh into router, meaning router is crashed + CheckSshAnswer answer = (CheckSshAnswer) cmds.getAnswer("checkSsh"); + if (answer == null || !answer.getResult()) { + return true; + } + + return false; + } +}