From ce1659e8fcce61e0a6e7c75190af668f75172c87 Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Mon, 30 Oct 2023 19:37:12 -0400 Subject: [PATCH 1/2] NSX: Create and Delete static NAT & Port forward rules (#8131) * NSX: Create and delete NSX Static Nat rules * fix issues with static nat * add static nat * Support to add and delete Port forward rules * add license * fix adding multiple pf rules * cleanup * fix lint check * fix smoke tests * fix smoke tests --- .../network/CreateNetworkOfferingCmd.java | 5 +- .../admin/vpc/CreateVPCOfferingCmd.java | 4 +- .../api/CreateNsxPortForwardRuleCommand.java | 52 ++++ .../agent/api/CreateNsxStaticNatCommand.java | 25 ++ .../agent/api/DeleteNsxNatRuleCommand.java | 54 ++++ .../agent/api/NsxNetworkCommand.java | 112 ++++++++ .../cloudstack/resource/NsxNetworkRule.java | 241 ++++++++++++++++++ .../cloudstack/resource/NsxResource.java | 64 +++++ .../cloudstack/service/NsxApiClient.java | 176 +++++++++++++ .../apache/cloudstack/service/NsxElement.java | 144 ++++++++++- .../cloudstack/service/NsxServiceImpl.java | 44 ++++ .../cloudstack/utils/NsxControllerUtils.java | 10 + .../cloud/server/ConfigurationServerImpl.java | 3 + 13 files changed, 929 insertions(+), 5 deletions(-) create mode 100644 plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/agent/api/CreateNsxPortForwardRuleCommand.java create mode 100644 plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/agent/api/CreateNsxStaticNatCommand.java create mode 100644 plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/agent/api/DeleteNsxNatRuleCommand.java create mode 100644 plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/agent/api/NsxNetworkCommand.java create mode 100644 plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/resource/NsxNetworkRule.java diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/network/CreateNetworkOfferingCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/network/CreateNetworkOfferingCmd.java index e12ab53c073..1ba182d4d61 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/network/CreateNetworkOfferingCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/network/CreateNetworkOfferingCmd.java @@ -23,6 +23,7 @@ import java.util.Iterator; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; @@ -235,7 +236,7 @@ public class CreateNetworkOfferingCmd extends BaseCmd { } public List getSupportedServices() { - if (!forNsx) { + if (!isForNsx()) { return supportedServices == null ? new ArrayList() : supportedServices; } else { List services = new ArrayList<>(List.of( @@ -283,7 +284,7 @@ public class CreateNetworkOfferingCmd extends BaseCmd { } public Boolean isForNsx() { - return forNsx; + return !Objects.isNull(forNsx) && forNsx; } public String getNsxMode() { diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/vpc/CreateVPCOfferingCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/vpc/CreateVPCOfferingCmd.java index c55a4571733..a88b2845d93 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/vpc/CreateVPCOfferingCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/vpc/CreateVPCOfferingCmd.java @@ -145,10 +145,10 @@ public class CreateVPCOfferingCmd extends BaseAsyncCreateCmd { } public List getSupportedServices() { - if (!forNsx && CollectionUtils.isEmpty(supportedServices)) { + if (!isForNsx() && CollectionUtils.isEmpty(supportedServices)) { throw new InvalidParameterValueException("Supported services needs to be provided"); } - if (forNsx) { + if (isForNsx()) { return List.of( Dhcp.getName(), Dns.getName(), diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/agent/api/CreateNsxPortForwardRuleCommand.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/agent/api/CreateNsxPortForwardRuleCommand.java new file mode 100644 index 00000000000..96d5213abd5 --- /dev/null +++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/agent/api/CreateNsxPortForwardRuleCommand.java @@ -0,0 +1,52 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.agent.api; + +public class CreateNsxPortForwardRuleCommand extends NsxNetworkCommand { + private final String publicPort; + private final String privatePort; + private final String protocol; + private final long ruleId; + + + public CreateNsxPortForwardRuleCommand(long domainId, long accountId, long zoneId, Long networkResourceId, + String networkResourceName, boolean isResourceVpc, Long vmId, + long ruleId, String publicIp, String vmIp, String publicPort, String privatePort, String protocol) { + super(domainId, accountId, zoneId, networkResourceId, networkResourceName, isResourceVpc, vmId, publicIp, vmIp); + this.publicPort = publicPort; + this.privatePort = privatePort; + this.ruleId = ruleId; + this.protocol = protocol; + + } + + public String getPublicPort() { + return publicPort; + } + + public String getPrivatePort() { + return privatePort; + } + + public long getRuleId() { + return ruleId; + } + + public String getProtocol() { + return protocol; + } +} diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/agent/api/CreateNsxStaticNatCommand.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/agent/api/CreateNsxStaticNatCommand.java new file mode 100644 index 00000000000..08c13420d1d --- /dev/null +++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/agent/api/CreateNsxStaticNatCommand.java @@ -0,0 +1,25 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.agent.api; + +public class CreateNsxStaticNatCommand extends NsxNetworkCommand { + + public CreateNsxStaticNatCommand(long domainId, long accountId, long zoneId, Long networkResourceId, String networkResourceName, + boolean isResourceVpc, Long vmId, String publicIp, String vmIp) { + super(domainId, accountId, zoneId, networkResourceId, networkResourceName, isResourceVpc, vmId, publicIp, vmIp); + } +} diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/agent/api/DeleteNsxNatRuleCommand.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/agent/api/DeleteNsxNatRuleCommand.java new file mode 100644 index 00000000000..82ca54d5b0d --- /dev/null +++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/agent/api/DeleteNsxNatRuleCommand.java @@ -0,0 +1,54 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.agent.api; + +import com.cloud.network.Network; + +public class DeleteNsxNatRuleCommand extends NsxNetworkCommand { + private Long ruleId; + private Network.Service service; + + private String privatePort; + private String protocol; + public DeleteNsxNatRuleCommand(long domainId, long accountId, long zoneId, Long networkResourceId, String networkResourceName, + boolean isResourceVpc, Long vmId, Long ruleId, String privatePort, String protocol) { + super(domainId, accountId, zoneId, networkResourceId, networkResourceName, isResourceVpc, vmId); + this.ruleId = ruleId; + this.privatePort = privatePort; + this.protocol = protocol; + } + + public Long getRuleId() { + return ruleId; + } + + public Network.Service getService() { + return service; + } + + public void setService(Network.Service service) { + this.service = service; + } + + public String getPrivatePort() { + return privatePort; + } + + public String getProtocol() { + return protocol; + } +} diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/agent/api/NsxNetworkCommand.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/agent/api/NsxNetworkCommand.java new file mode 100644 index 00000000000..e5f49af9c17 --- /dev/null +++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/agent/api/NsxNetworkCommand.java @@ -0,0 +1,112 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.agent.api; + +import java.util.Objects; + +public class NsxNetworkCommand extends NsxCommand { + private Long networkResourceId; + private String networkResourceName; + private boolean isResourceVpc; + private Long vmId; + private String publicIp; + private String vmIp; + + public NsxNetworkCommand(long domainId, long accountId, long zoneId, Long networkResourceId, String networkResourceName, + boolean isResourceVpc, Long vmId, String publicIp, String vmIp) { + super(domainId, accountId, zoneId); + this.networkResourceId = networkResourceId; + this.networkResourceName = networkResourceName; + this.isResourceVpc = isResourceVpc; + this.vmId = vmId; + this.publicIp = publicIp; + this.vmIp = vmIp; + } + + public NsxNetworkCommand(long domainId, long accountId, long zoneId, Long networkResourceId, String networkResourceName, + boolean isResourceVpc, Long vmId) { + super(domainId, accountId, zoneId); + this.networkResourceId = networkResourceId; + this.networkResourceName = networkResourceName; + this.isResourceVpc = isResourceVpc; + this.vmId = vmId; + } + + public Long getNetworkResourceId() { + return networkResourceId; + } + + public void setNetworkResourceId(long networkResourceId) { + this.networkResourceId = networkResourceId; + } + + public String getNetworkResourceName() { + return networkResourceName; + } + + public void setNetworkResourceName(String networkResourceName) { + this.networkResourceName = networkResourceName; + } + + public boolean isResourceVpc() { + return isResourceVpc; + } + + public void setResourceVpc(boolean resourceVpc) { + isResourceVpc = resourceVpc; + } + + public Long getVmId() { + return vmId; + } + + public void setVmId(Long vmId) { + this.vmId = vmId; + } + + public String getPublicIp() { + return publicIp; + } + + public void setPublicIp(String publicIp) { + this.publicIp = publicIp; + } + + public String getVmIp() { + return vmIp; + } + + public void setVmIp(String vmIp) { + this.vmIp = vmIp; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + if (!super.equals(o)) return false; + NsxNetworkCommand that = (NsxNetworkCommand) o; + return networkResourceId == that.networkResourceId && vmId == that.vmId && + Objects.equals(networkResourceName, that.networkResourceName) && Objects.equals(publicIp, that.publicIp) + && Objects.equals(vmIp, that.vmIp); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), networkResourceId, networkResourceName, vmId, publicIp, vmIp); + } +} diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/resource/NsxNetworkRule.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/resource/NsxNetworkRule.java new file mode 100644 index 00000000000..fc670429a19 --- /dev/null +++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/resource/NsxNetworkRule.java @@ -0,0 +1,241 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.resource; + +public class NsxNetworkRule { + private long domainId; + private long accountId; + private long zoneId; + private Long networkResourceId; + private String networkResourceName; + private boolean isVpcResource; + private long vmId; + private long ruleId; + private String publicIp; + private String vmIp; + private String publicPort; + private String privatePort; + private String protocol; + + public long getDomainId() { + return domainId; + } + + public void setDomainId(long domainId) { + this.domainId = domainId; + } + + public long getAccountId() { + return accountId; + } + + public void setAccountId(long accountId) { + this.accountId = accountId; + } + + public long getZoneId() { + return zoneId; + } + + public void setZoneId(long zoneId) { + this.zoneId = zoneId; + } + + public Long getNetworkResourceId() { + return networkResourceId; + } + + public void setNetworkResourceId(Long networkResourceId) { + this.networkResourceId = networkResourceId; + } + + public String getNetworkResourceName() { + return networkResourceName; + } + + public void setNetworkResourceName(String networkResourceName) { + this.networkResourceName = networkResourceName; + } + + public boolean isVpcResource() { + return isVpcResource; + } + + public void setVpcResource(boolean vpcResource) { + isVpcResource = vpcResource; + } + + public long getVmId() { + return vmId; + } + + public void setVmId(long vmId) { + this.vmId = vmId; + } + + public long getRuleId() { + return ruleId; + } + + public void setRuleId(long ruleId) { + this.ruleId = ruleId; + } + + public String getPublicIp() { + return publicIp; + } + + public void setPublicIp(String publicIp) { + this.publicIp = publicIp; + } + + public String getVmIp() { + return vmIp; + } + + public void setVmIp(String vmIp) { + this.vmIp = vmIp; + } + + public String getPublicPort() { + return publicPort; + } + + public void setPublicPort(String publicPort) { + this.publicPort = publicPort; + } + + public String getPrivatePort() { + return privatePort; + } + + public void setPrivatePort(String privatePort) { + this.privatePort = privatePort; + } + + public String getProtocol() { + return protocol; + } + + public void setProtocol(String protocol) { + this.protocol = protocol; + } + + public static final class Builder { + private long domainId; + private long accountId; + private long zoneId; + private Long networkResourceId; + private String networkResourceName; + private boolean isVpcResource; + private long vmId; + + private long ruleId; + private String publicIp; + private String vmIp; + private String publicPort; + private String privatePort; + private String protocol; + + public Builder() { + } + + public Builder setDomainId(long domainId) { + this.domainId = domainId; + return this; + } + + public Builder setAccountId(long accountId) { + this.accountId = accountId; + return this; + } + + public Builder setZoneId(long zoneId) { + this.zoneId = zoneId; + return this; + } + + public Builder setNetworkResourceId(Long networkResourceId) { + this.networkResourceId = networkResourceId; + return this; + } + + public Builder setNetworkResourceName(String networkResourceName) { + this.networkResourceName = networkResourceName; + return this; + } + + public Builder setVpcResource(boolean isVpcResource) { + this.isVpcResource = isVpcResource; + return this; + } + + + public Builder setVmId(long vmId) { + this.vmId = vmId; + return this; + } + + public Builder setRuleId(long ruleId) { + this.ruleId = ruleId; + return this; + } + + public Builder setPublicIp(String publicIp) { + this.publicIp = publicIp; + return this; + } + + public Builder setVmIp(String vmIp) { + this.vmIp = vmIp; + return this; + } + + public Builder setPublicPort(String publicPort) { + this.publicPort = publicPort; + return this; + } + + public Builder setPrivatePort(String privatePort) { + this.privatePort = privatePort; + return this; + } + + public Builder setProtocol(String protocol) { + this.protocol = protocol; + return this; + } + + public NsxNetworkRule build() { + NsxNetworkRule rule = new NsxNetworkRule(); + rule.setDomainId(this.domainId); + rule.setAccountId(this.accountId); + rule.setZoneId(this.zoneId); + rule.setNetworkResourceId(this.networkResourceId); + rule.setNetworkResourceName(this.networkResourceName); + rule.setVpcResource(this.isVpcResource); + rule.setVmId(this.vmId); + rule.setVmIp(this.vmIp); + rule.setPublicIp(this.publicIp); + rule.setPublicPort(this.publicPort); + rule.setPrivatePort(this.privatePort); + rule.setProtocol(this.protocol); + rule.setRuleId(this.ruleId); + return rule; + } + } +} diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/resource/NsxResource.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/resource/NsxResource.java index 46e14e77121..2a375bcd483 100644 --- a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/resource/NsxResource.java +++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/resource/NsxResource.java @@ -24,6 +24,7 @@ import com.cloud.agent.api.ReadyAnswer; import com.cloud.agent.api.ReadyCommand; import com.cloud.agent.api.StartupCommand; import com.cloud.host.Host; +import com.cloud.network.Network; import com.cloud.resource.ServerResource; import com.cloud.utils.exception.CloudRuntimeException; @@ -35,9 +36,12 @@ import com.vmware.nsx_policy.model.SiteListResult; import org.apache.cloudstack.NsxAnswer; import org.apache.cloudstack.StartupNsxCommand; import org.apache.cloudstack.agent.api.CreateNsxDhcpRelayConfigCommand; +import org.apache.cloudstack.agent.api.CreateNsxPortForwardRuleCommand; import org.apache.cloudstack.agent.api.CreateNsxSegmentCommand; +import org.apache.cloudstack.agent.api.CreateNsxStaticNatCommand; import org.apache.cloudstack.agent.api.CreateNsxTier1GatewayCommand; import org.apache.cloudstack.agent.api.DeleteNsxSegmentCommand; +import org.apache.cloudstack.agent.api.DeleteNsxNatRuleCommand; import org.apache.cloudstack.agent.api.DeleteNsxTier1GatewayCommand; import org.apache.cloudstack.service.NsxApiClient; import org.apache.cloudstack.utils.NsxControllerUtils; @@ -104,6 +108,12 @@ public class NsxResource implements ServerResource { return executeRequest((CreateNsxTier1GatewayCommand) cmd); } else if (cmd instanceof CreateNsxDhcpRelayConfigCommand) { return executeRequest((CreateNsxDhcpRelayConfigCommand) cmd); + } else if (cmd instanceof CreateNsxStaticNatCommand) { + return executeRequest((CreateNsxStaticNatCommand) cmd); + } else if (cmd instanceof DeleteNsxNatRuleCommand) { + return executeRequest((DeleteNsxNatRuleCommand) cmd); + } else if (cmd instanceof CreateNsxPortForwardRuleCommand) { + return executeRequest((CreateNsxPortForwardRuleCommand) cmd); } else { return Answer.createUnsupportedCommandAnswer(cmd); } @@ -337,6 +347,60 @@ public class NsxResource implements ServerResource { return new NsxAnswer(cmd, true, null); } + private NsxAnswer executeRequest(CreateNsxStaticNatCommand cmd) { + String staticNatRuleName = NsxControllerUtils.getStaticNatRuleName(cmd.getDomainId(), cmd.getAccountId(), cmd.getZoneId(), + cmd.getNetworkResourceId(), cmd.isResourceVpc()); + String tier1GatewayName = NsxControllerUtils.getTier1GatewayName(cmd.getDomainId(), cmd.getAccountId(), cmd.getZoneId(), + cmd.getNetworkResourceId(), cmd.isResourceVpc()); + try { + nsxApiClient.createStaticNatRule(cmd.getNetworkResourceName(), tier1GatewayName, staticNatRuleName, cmd.getPublicIp(), cmd.getVmIp()); + } catch (Exception e) { + LOGGER.error(String.format("Failed to add NSX static NAT rule %s for network: %s", staticNatRuleName, cmd.getNetworkResourceName())); + return new NsxAnswer(cmd, new CloudRuntimeException(e.getMessage())); + } + return new NsxAnswer(cmd, true, null); + } + + private NsxAnswer executeRequest(CreateNsxPortForwardRuleCommand cmd) { + String ruleName = NsxControllerUtils.getPortForwardRuleName(cmd.getDomainId(), cmd.getAccountId(), cmd.getZoneId(), + cmd.getNetworkResourceId(), cmd.getRuleId(), cmd.isResourceVpc()); + String tier1GatewayName = NsxControllerUtils.getTier1GatewayName(cmd.getDomainId(), cmd.getAccountId(), cmd.getZoneId(), + cmd.getNetworkResourceId(), cmd.isResourceVpc()); + try { + String privatePort = cmd.getPrivatePort(); + String service = privatePort.contains("-") ? nsxApiClient.createNsxInfraService(ruleName, privatePort, cmd.getProtocol()) : + nsxApiClient.getNsxInfraServices(ruleName, privatePort, cmd.getProtocol()); + + nsxApiClient.createPortForwardingRule(ruleName, tier1GatewayName, cmd.getNetworkResourceName(), cmd.getPublicIp(), + cmd.getVmIp(), cmd.getPublicPort(), service); + } catch (Exception e) { + LOGGER.error(String.format("Failed to add NSX port forward rule %s for network: %s", ruleName, cmd.getNetworkResourceName())); + return new NsxAnswer(cmd, new CloudRuntimeException(e.getMessage())); + } + return new NsxAnswer(cmd, true, null); + } + + private NsxAnswer executeRequest(DeleteNsxNatRuleCommand cmd) { + String ruleName = null; + if (cmd.getService() == Network.Service.StaticNat) { + ruleName = NsxControllerUtils.getStaticNatRuleName(cmd.getDomainId(), cmd.getAccountId(), cmd.getZoneId(), + cmd.getNetworkResourceId(), cmd.isResourceVpc()); + } else if (cmd.getService() == Network.Service.PortForwarding) { + ruleName = NsxControllerUtils.getPortForwardRuleName(cmd.getDomainId(), cmd.getAccountId(), cmd.getZoneId(), + cmd.getNetworkResourceId(), cmd.getRuleId(), cmd.isResourceVpc()); + } + String tier1GatewayName = NsxControllerUtils.getTier1GatewayName(cmd.getDomainId(), cmd.getAccountId(), cmd.getZoneId(), + cmd.getNetworkResourceId(), cmd.isResourceVpc()); + try { + nsxApiClient.deleteNatRule(cmd.getService(), cmd.getPrivatePort(), cmd.getProtocol(), + cmd.getNetworkResourceName(), tier1GatewayName, ruleName); + } catch (Exception e) { + LOGGER.error(String.format("Failed to add NSX static NAT rule %s for network: %s", ruleName, cmd.getNetworkResourceName())); + return new NsxAnswer(cmd, new CloudRuntimeException(e.getMessage())); + } + return new NsxAnswer(cmd, true, null); + } + @Override public boolean start() { return true; diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxApiClient.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxApiClient.java index 0cdc55cb3f0..534030a6211 100644 --- a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxApiClient.java +++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxApiClient.java @@ -17,21 +17,27 @@ package org.apache.cloudstack.service; import com.cloud.exception.InvalidParameterValueException; +import com.cloud.network.Network; import com.cloud.utils.exception.CloudRuntimeException; import com.vmware.nsx.model.TransportZone; import com.vmware.nsx.model.TransportZoneListResult; import com.vmware.nsx_policy.infra.DhcpRelayConfigs; import com.vmware.nsx_policy.infra.Segments; +import com.vmware.nsx_policy.infra.Services; import com.vmware.nsx_policy.infra.Sites; import com.vmware.nsx_policy.infra.Tier1s; import com.vmware.nsx_policy.infra.sites.EnforcementPoints; import com.vmware.nsx_policy.infra.tier_0s.LocaleServices; +import com.vmware.nsx_policy.infra.tier_1s.nat.NatRules; import com.vmware.nsx_policy.model.ApiError; import com.vmware.nsx_policy.model.DhcpRelayConfig; import com.vmware.nsx_policy.model.EnforcementPointListResult; +import com.vmware.nsx_policy.model.L4PortSetServiceEntry; import com.vmware.nsx_policy.model.LocaleServicesListResult; +import com.vmware.nsx_policy.model.PolicyNatRule; import com.vmware.nsx_policy.model.Segment; import com.vmware.nsx_policy.model.SegmentSubnet; +import com.vmware.nsx_policy.model.ServiceListResult; import com.vmware.nsx_policy.model.SiteListResult; import com.vmware.nsx_policy.model.Tier1; import com.vmware.vapi.bindings.Service; @@ -46,10 +52,13 @@ import com.vmware.vapi.internal.protocol.client.rest.authn.BasicAuthenticationAp import com.vmware.vapi.protocol.HttpConfiguration; import com.vmware.vapi.std.errors.Error; import org.apache.cloudstack.utils.NsxControllerUtils; +import org.apache.commons.collections.CollectionUtils; import org.apache.log4j.Logger; import java.util.List; +import java.util.Objects; import java.util.function.Function; +import java.util.stream.Collectors; public class NsxApiClient { @@ -77,6 +86,16 @@ public class NsxApiClient { private enum TransportType { OVERLAY, VLAN } + private enum NatId { USER, INTERNAL, DEFAULT } + + private enum NatAction {SNAT, DNAT, REFLEXIVE} + + private enum FirewallMatch { + MATCH_INTERNAL_ADDRESS, + MATCH_EXTERNAL_ADDRESS, + BYPASS + } + public enum RouteAdvertisementType { TIER1_STATIC_ROUTES, TIER1_CONNECTED, TIER1_NAT, TIER1_LB_VIP, TIER1_LB_SNAT, TIER1_DNS_FORWARDER_IP, TIER1_IPSEC_LOCAL_ENDPOINT } @@ -288,4 +307,161 @@ public class NsxApiClient { throw new CloudRuntimeException(msg); } } + + public void createStaticNatRule(String vpcName, String tier1GatewayName, + String ruleName, String publicIp, String vmIp) { + try { + NatRules natService = (NatRules) nsxService.apply(NatRules.class); + PolicyNatRule rule = new PolicyNatRule.Builder() + .setId(ruleName) + .setDisplayName(ruleName) + .setAction(NatAction.DNAT.name()) + .setFirewallMatch(FirewallMatch.MATCH_INTERNAL_ADDRESS.name()) + .setDestinationNetwork(publicIp) + .setTranslatedNetwork(vmIp) + .setEnabled(true) + .build(); + + LOGGER.debug(String.format("Creating NSX static NAT rule %s for tier-1 gateway %s (VPC: %s)", ruleName, tier1GatewayName, vpcName)); + natService.patch(tier1GatewayName, NatId.USER.name(), ruleName, rule); + } catch (Error error) { + ApiError ae = error.getData()._convertTo(ApiError.class); + String msg = String.format("Error creating NSX Static NAT rule %s for tier-1 gateway %s (VPC: %s), due to %s", + ruleName, tier1GatewayName, vpcName, ae.getErrorMessage()); + LOGGER.error(msg); + throw new CloudRuntimeException(msg); + } + } + + public void deleteNatRule(Network.Service service, String privatePort, String protocol, String networkName, String tier1GatewayName, String ruleName) { + try { + NatRules natService = (NatRules) nsxService.apply(NatRules.class); + LOGGER.debug(String.format("Deleting NSX static NAT rule %s for tier-1 gateway %s (network: %s)", ruleName, tier1GatewayName, networkName)); + // delete NAT rule + natService.delete(tier1GatewayName, NatId.USER.name(), ruleName); + if (service == Network.Service.PortForwarding) { + String svcName = getServiceName(ruleName, privatePort, protocol); + // Delete service + Services services = (Services) nsxService.apply(Services.class); + services.delete(svcName); + } + } catch (Error error) { + ApiError ae = error.getData()._convertTo(ApiError.class); + String msg = String.format("Failed to delete NSX Static NAT rule %s for tier-1 gateway %s (VPC: %s), due to %s", + ruleName, tier1GatewayName, networkName, ae.getErrorMessage()); + LOGGER.error(msg); + throw new CloudRuntimeException(msg); + } + } + + public void createPortForwardingRule(String ruleName, String tier1GatewayName, String networkName, String publicIp, + String vmIp, String publicPort, String service) { + try { + NatRules natService = (NatRules) nsxService.apply(NatRules.class); + LOGGER.debug(String.format("Creating NSX Port-Forwarding NAT %s for network %s", ruleName, networkName)); + PolicyNatRule rule = new PolicyNatRule.Builder() + .setId(ruleName) + .setDisplayName(ruleName) + .setAction(NatAction.DNAT.name()) + .setFirewallMatch(FirewallMatch.MATCH_INTERNAL_ADDRESS.name()) + .setDestinationNetwork(publicIp) + .setTranslatedNetwork(vmIp) + .setTranslatedPorts(String.valueOf(publicPort)) + .setService(service) + .setEnabled(true) + .build(); + natService.patch(tier1GatewayName, NatId.USER.name(), ruleName, rule); + } catch (Error error) { + ApiError ae = error.getData()._convertTo(ApiError.class); + String msg = String.format("Failed to delete NSX Port-forward rule %s for network: %s, due to %s", + ruleName, networkName, ae.getErrorMessage()); + LOGGER.error(msg); + throw new CloudRuntimeException(msg); + } + } + + public String getNsxInfraServices(String ruleName, String port, String protocol) { + try { + Services service = (Services) nsxService.apply(Services.class); + + // Find default service if present + ServiceListResult serviceList = service.list(null, true, false, null, null, null, null); + + List services = serviceList.getResults(); + List matchedDefaultSvc = services.parallelStream().filter(svc -> + (svc.getServiceEntries().get(0) instanceof L4PortSetServiceEntry) && + ((L4PortSetServiceEntry) svc.getServiceEntries().get(0)).getDestinationPorts().get(0).equals(port) + && (((L4PortSetServiceEntry) svc.getServiceEntries().get(0)).getL4Protocol().equals(protocol))) + .map(svc -> ((L4PortSetServiceEntry) svc.getServiceEntries().get(0)).getDestinationPorts().get(0)) + .collect(Collectors.toList()); + if (!CollectionUtils.isEmpty(matchedDefaultSvc)) { + return matchedDefaultSvc.get(0); + } + + // Else, find if there's a service matching the rule name + String servicePath = getServiceById(ruleName); + if (Objects.nonNull(servicePath)) { + return servicePath; + } + + // Else, create a service entry + return createNsxInfraService(ruleName, port, protocol); + } catch (Error error) { + ApiError ae = error.getData()._convertTo(ApiError.class); + String msg = String.format("Failed to list NSX infra service, due to: %s", ae.getErrorMessage()); + LOGGER.error(msg); + throw new CloudRuntimeException(msg); + } + } + + public String createNsxInfraService(String ruleName, String port, String protocol) { + try { + String serviceEntryName = getServiceEntryName(ruleName, port, protocol); + String serviceName = getServiceName(ruleName, port, protocol); + Services service = (Services) nsxService.apply(Services.class); + com.vmware.nsx_policy.model.Service infraService = new com.vmware.nsx_policy.model.Service.Builder() + .setServiceEntries(List.of( + new L4PortSetServiceEntry.Builder() + .setId(serviceEntryName) + .setDisplayName(serviceEntryName) + .setDestinationPorts(List.of(port)) + .setL4Protocol(protocol) + .build() + )) + .setId(serviceName) + .setDisplayName(serviceName) + .build(); + service.patch(serviceName, infraService); + + com.vmware.nsx_policy.model.Service svc = service.get(serviceName); + return svc.getServiceEntries().get(0)._getDataValue().getField("parent_path").toString(); + + } catch (Error error) { + ApiError ae = error.getData()._convertTo(ApiError.class); + String msg = String.format("Failed to create NSX infra service, due to: %s", ae.getErrorMessage()); + LOGGER.error(msg); + throw new CloudRuntimeException(msg); + } + } + + private String getServiceById(String ruleName) { + try { + Services service = (Services) nsxService.apply(Services.class); + com.vmware.nsx_policy.model.Service svc1 = service.get(ruleName); + if (Objects.nonNull(svc1)) { + return ((L4PortSetServiceEntry) svc1.getServiceEntries().get(0)).getParentPath(); + } + } catch (Exception e) { + return null; + } + return null; + } + + private String getServiceName(String ruleName, String port, String protocol) { + return ruleName + "-SVC-" + port + "-" +protocol; + } + + private String getServiceEntryName(String ruleName, String port, String protocol) { + return ruleName + "-SE-" + port + "-" + protocol; + } } diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxElement.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxElement.java index 846875ad698..c8ac091485b 100644 --- a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxElement.java +++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxElement.java @@ -24,6 +24,7 @@ import com.cloud.agent.api.AgentControlCommand; import com.cloud.agent.api.Answer; import com.cloud.agent.api.Command; import com.cloud.agent.api.StartupCommand; +import com.cloud.api.ApiDBUtils; import com.cloud.dc.DataCenterVO; import com.cloud.dc.dao.DataCenterDao; import com.cloud.deploy.DeployDestination; @@ -41,17 +42,28 @@ import com.cloud.network.Network; import com.cloud.network.NetworkModel; import com.cloud.network.Networks; import com.cloud.network.PhysicalNetworkServiceProvider; +import com.cloud.network.PublicIpAddress; +import com.cloud.network.dao.IPAddressDao; +import com.cloud.network.dao.IPAddressVO; import com.cloud.network.dao.NetworkDao; import com.cloud.network.dao.NetworkVO; import com.cloud.network.dao.PhysicalNetworkDao; import com.cloud.network.dao.PhysicalNetworkVO; import com.cloud.network.element.DhcpServiceProvider; import com.cloud.network.element.DnsServiceProvider; +import com.cloud.network.element.IpDeployer; +import com.cloud.network.element.PortForwardingServiceProvider; +import com.cloud.network.element.StaticNatServiceProvider; import com.cloud.network.element.VpcProvider; +import com.cloud.network.rules.FirewallRule; +import com.cloud.network.rules.PortForwardingRule; +import com.cloud.network.rules.StaticNat; import com.cloud.network.vpc.NetworkACLItem; import com.cloud.network.vpc.PrivateGateway; import com.cloud.network.vpc.StaticRouteProfile; import com.cloud.network.vpc.Vpc; +import com.cloud.network.vpc.VpcVO; +import com.cloud.network.vpc.dao.VpcDao; import com.cloud.offering.NetworkOffering; import com.cloud.resource.ResourceManager; import com.cloud.resource.ResourceStateAdapter; @@ -59,14 +71,19 @@ import com.cloud.resource.ServerResource; import com.cloud.resource.UnableDeleteHostException; import com.cloud.user.Account; import com.cloud.user.AccountManager; +import com.cloud.uservm.UserVm; import com.cloud.utils.Pair; import com.cloud.utils.component.AdapterBase; import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.vm.Nic; import com.cloud.vm.NicProfile; import com.cloud.vm.ReservationContext; +import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachineProfile; +import com.cloud.vm.dao.VMInstanceDao; import net.sf.ehcache.config.InvalidConfigurationException; import org.apache.cloudstack.StartupNsxCommand; +import org.apache.cloudstack.resource.NsxNetworkRule; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -74,6 +91,7 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Set; @@ -81,7 +99,7 @@ import java.util.function.LongFunction; @Component public class NsxElement extends AdapterBase implements DhcpServiceProvider, DnsServiceProvider, VpcProvider, - ResourceStateAdapter, Listener { + StaticNatServiceProvider, IpDeployer, PortForwardingServiceProvider, ResourceStateAdapter, Listener { @Inject AccountManager accountMgr; @@ -101,6 +119,12 @@ public class NsxElement extends AdapterBase implements DhcpServiceProvider, DnsS NetworkModel networkModel; @Inject DomainDao domainDao; + @Inject + IPAddressDao ipAddressDao; + @Inject + VMInstanceDao vmInstanceDao; + @Inject + VpcDao vpcDao; private static final Logger LOGGER = Logger.getLogger(NsxElement.class); @@ -172,6 +196,11 @@ public class NsxElement extends AdapterBase implements DhcpServiceProvider, DnsS return capabilities; } + @Override + public boolean applyIps(Network network, List ipAddress, Set services) throws ResourceUnavailableException { + return true; + } + @Override public Network.Provider getProvider() { return Network.Provider.Nsx; @@ -416,4 +445,117 @@ public class NsxElement extends AdapterBase implements DhcpServiceProvider, DnsS } private final LongFunction zoneFunction = zoneId -> dataCenterDao.findById(zoneId); + + @Override + public IpDeployer getIpDeployer(Network network) { + return this; + } + + @Override + public boolean applyStaticNats(Network config, List rules) throws ResourceUnavailableException { + for(StaticNat staticNat : rules) { + long sourceIpAddressId = staticNat.getSourceIpAddressId(); + IPAddressVO ipAddressVO = ipAddressDao.findByIdIncludingRemoved(sourceIpAddressId); + VMInstanceVO vm = vmInstanceDao.findByIdIncludingRemoved(ipAddressVO.getAssociatedWithVmId()); + // floating ip is released when nic was deleted + if (vm == null || networkModel.getNicInNetworkIncludingRemoved(vm.getId(), config.getId()) == null) { + continue; + } + Nic nic = networkModel.getNicInNetworkIncludingRemoved(vm.getId(), config.getId()); + Network publicNetwork = networkModel.getSystemNetworkByZoneAndTrafficType(config.getDataCenterId(), Networks.TrafficType.Public); + Pair vpcOrNetwork = getVpcOrNetwork(config.getVpcId(), config.getId()); + VpcVO vpc = vpcOrNetwork.first(); + NetworkVO network = vpcOrNetwork.second(); + Long networkResourceId = Objects.nonNull(vpc) ? vpc.getId() : network.getId(); + String networkResourceName = Objects.nonNull(vpc) ? vpc.getName() : network.getName(); + boolean isVpcResource = Objects.nonNull(vpc); + if (!staticNat.isForRevoke()) { + return nsxService.createStaticNatRule(config.getDataCenterId(), config.getDomainId(), config.getAccountId(), + networkResourceId, networkResourceName, isVpcResource, vm.getId(), + ipAddressVO.getAddress().addr(), staticNat.getDestIpAddress()); + } else { + return nsxService.deleteStaticNatRule(config.getDataCenterId(), config.getDomainId(), config.getAccountId(), + networkResourceId, networkResourceName, isVpcResource); + } + } + return false; + } + + @Override + public boolean applyPFRules(Network network, List rules) throws ResourceUnavailableException { + if (!canHandle(network, Network.Service.PortForwarding)) { + return false; + } + for (PortForwardingRule rule : rules) { + IPAddressVO publicIp = ApiDBUtils.findIpAddressById(rule.getSourceIpAddressId()); + UserVm vm = ApiDBUtils.findUserVmById(rule.getVirtualMachineId()); + if (vm == null || networkModel.getNicInNetwork(vm.getId(), network.getId()) == null) { + continue; + } + Pair vpcOrNetwork = getVpcOrNetwork(network.getVpcId(), network.getId()); + VpcVO vpc = vpcOrNetwork.first(); + NetworkVO networkVO = vpcOrNetwork.second(); + Long networkResourceId = Objects.nonNull(vpc) ? vpc.getId() : networkVO.getId(); + String networkResourceName = Objects.nonNull(vpc) ? vpc.getName() : networkVO.getName(); + boolean isVpcResource = Objects.nonNull(vpc); + long domainId = Objects.nonNull(vpc) ? vpc.getDomainId() : networkVO.getDomainId(); + long accountId = Objects.nonNull(vpc) ? vpc.getAccountId() : networkVO.getAccountId(); + long zoneId = Objects.nonNull(vpc) ? vpc.getZoneId() : networkVO.getDataCenterId(); + String publicPort = getPublicPortRange(rule); + + String privatePort = getPrivatePortRange(rule); + + // TODO: add builder to reduce signature params ; should we pass port range? + NsxNetworkRule networkRule = new NsxNetworkRule.Builder() + .setDomainId(domainId) + .setAccountId(accountId) + .setZoneId(zoneId) + .setNetworkResourceId(networkResourceId) + .setNetworkResourceName(networkResourceName) + .setVpcResource(isVpcResource) + .setVmId(vm.getId()) + .setVmIp(vm.getPrivateIpAddress()) + .setPublicIp(publicIp.getAddress().addr()) + .setPrivatePort(privatePort) + .setPublicPort(publicPort) + .setRuleId(rule.getId()) + .setProtocol(rule.getProtocol().toUpperCase(Locale.ROOT)) + .build(); + if (rule.getState() == FirewallRule.State.Add) { + return nsxService.createPortForwardRule(networkRule); + } else if (rule.getState() == FirewallRule.State.Revoke) { + return nsxService.deletePortForwardRule(networkRule); + } + } + return true; + } + + public Pair getVpcOrNetwork(Long vpcId, long networkId) { + VpcVO vpc = null; + NetworkVO network = null; + if (Objects.nonNull(vpcId)) { + vpc = vpcDao.findById(vpcId); + if (Objects.isNull(vpc)) { + throw new CloudRuntimeException(String.format("Failed to find VPC with id: %s", vpcId)); + } + } else { + network = networkDao.findById(networkId); + if (Objects.isNull(network)) { + throw new CloudRuntimeException(String.format("Failed to find network with id: %s", networkId)); + } + } + return new Pair<>(vpc, network); + } + + private static String getPublicPortRange(PortForwardingRule rule) { + return rule.getDestinationPortStart() == rule.getDestinationPortEnd() ? + String.valueOf(rule.getDestinationPortStart()) : + String.valueOf(rule.getDestinationPortStart()).concat("-").concat(String.valueOf(rule.getDestinationPortEnd())); + } + + private static String getPrivatePortRange(PortForwardingRule rule) { + return Objects.equals(rule.getSourcePortStart(), rule.getSourcePortEnd()) ? + String.valueOf(rule.getSourcePortStart()) : + String.valueOf(rule.getSourcePortStart()).concat("-").concat(String.valueOf(rule.getSourcePortEnd())); + } } diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxServiceImpl.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxServiceImpl.java index 35b8220abd1..c2aef1404fb 100644 --- a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxServiceImpl.java +++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxServiceImpl.java @@ -16,14 +16,20 @@ // under the License. package org.apache.cloudstack.service; +import com.cloud.network.Network; +import com.cloud.network.dao.NetworkDao; import com.cloud.network.dao.NetworkVO; import com.cloud.network.vpc.VpcVO; import com.cloud.network.vpc.dao.VpcDao; import com.cloud.utils.exception.CloudRuntimeException; import org.apache.cloudstack.NsxAnswer; +import org.apache.cloudstack.agent.api.CreateNsxPortForwardRuleCommand; +import org.apache.cloudstack.agent.api.CreateNsxStaticNatCommand; import org.apache.cloudstack.agent.api.CreateNsxTier1GatewayCommand; import org.apache.cloudstack.agent.api.DeleteNsxSegmentCommand; +import org.apache.cloudstack.agent.api.DeleteNsxNatRuleCommand; import org.apache.cloudstack.agent.api.DeleteNsxTier1GatewayCommand; +import org.apache.cloudstack.resource.NsxNetworkRule; import org.apache.cloudstack.utils.NsxControllerUtils; import org.apache.log4j.Logger; @@ -35,6 +41,8 @@ public class NsxServiceImpl implements NsxService { NsxControllerUtils nsxControllerUtils; @Inject VpcDao vpcDao; + @Inject + NetworkDao networkDao; private static final Logger LOGGER = Logger.getLogger(NsxServiceImpl.class); @@ -80,4 +88,40 @@ public class NsxServiceImpl implements NsxService { } return result.getResult(); } + + public boolean createStaticNatRule(long zoneId, long domainId, long accountId, Long networkResourceId, String networkResourceName, + boolean isVpcResource, long vmId, String publicIp, String vmIp) { + CreateNsxStaticNatCommand createNsxStaticNatCommand = new CreateNsxStaticNatCommand(domainId, accountId, zoneId, + networkResourceId, networkResourceName, isVpcResource, vmId, publicIp, vmIp); + NsxAnswer result = nsxControllerUtils.sendNsxCommand(createNsxStaticNatCommand, zoneId); + return result.getResult(); + } + + public boolean deleteStaticNatRule(long zoneId, long domainId, long accountId, Long networkResourceId, String networkResourceName, + boolean isVpcResource) { + DeleteNsxNatRuleCommand deleteNsxStaticNatCommand = new DeleteNsxNatRuleCommand(domainId, accountId, zoneId, + networkResourceId, networkResourceName, isVpcResource, null, null, null, null); + deleteNsxStaticNatCommand.setService(Network.Service.StaticNat); + NsxAnswer result = nsxControllerUtils.sendNsxCommand(deleteNsxStaticNatCommand, zoneId); + return result.getResult(); + } + + public boolean createPortForwardRule(NsxNetworkRule netRule) { + // TODO: if port doesn't exist in default list of services, create a service entry + CreateNsxPortForwardRuleCommand createPortForwardCmd = new CreateNsxPortForwardRuleCommand(netRule.getDomainId(), + netRule.getAccountId(), netRule.getZoneId(), netRule.getNetworkResourceId(), + netRule.getNetworkResourceName(), netRule.isVpcResource(), netRule.getVmId(), netRule.getRuleId(), + netRule.getPublicIp(), netRule.getVmIp(), netRule.getPublicPort(), netRule.getPrivatePort(), netRule.getProtocol()); + NsxAnswer result = nsxControllerUtils.sendNsxCommand(createPortForwardCmd, netRule.getZoneId()); + return result.getResult(); + } + + public boolean deletePortForwardRule(NsxNetworkRule netRule) { + DeleteNsxNatRuleCommand deleteCmd = new DeleteNsxNatRuleCommand(netRule.getDomainId(), + netRule.getAccountId(), netRule.getZoneId(), netRule.getNetworkResourceId(), + netRule.getNetworkResourceName(), netRule.isVpcResource(), netRule.getVmId(), netRule.getRuleId(), netRule.getPrivatePort(), netRule.getPublicPort()); + deleteCmd.setService(Network.Service.PortForwarding); + NsxAnswer result = nsxControllerUtils.sendNsxCommand(deleteCmd, netRule.getZoneId()); + return result.getResult(); + } } diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/utils/NsxControllerUtils.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/utils/NsxControllerUtils.java index f26c7bf39ee..8c545476a6e 100644 --- a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/utils/NsxControllerUtils.java +++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/utils/NsxControllerUtils.java @@ -80,4 +80,14 @@ public class NsxControllerUtils { } return String.format("D%s-A%s-Z%s-V%s-S%s-%s", domainId, accountId, zoneId, vpcId, networkId, suffix); } + + public static String getStaticNatRuleName(long domainId, long accountId, long zoneId, Long networkResourceId, boolean isVpcResource) { + String suffix = "-STATICNAT"; + return getTier1GatewayName(domainId, accountId, zoneId, networkResourceId, isVpcResource) + suffix; + } + + public static String getPortForwardRuleName(long domainId, long accountId, long zoneId, Long networkResourceId, long ruleId, boolean isVpcResource) { + String suffix = "-PF"; + return getTier1GatewayName(domainId, accountId, zoneId, networkResourceId, isVpcResource) + suffix + ruleId; + } } diff --git a/server/src/main/java/com/cloud/server/ConfigurationServerImpl.java b/server/src/main/java/com/cloud/server/ConfigurationServerImpl.java index 4283236d260..031f622e30d 100644 --- a/server/src/main/java/com/cloud/server/ConfigurationServerImpl.java +++ b/server/src/main/java/com/cloud/server/ConfigurationServerImpl.java @@ -1247,6 +1247,9 @@ public class ConfigurationServerImpl extends ManagerBase implements Configuratio serviceProviderMap.put(Service.UserData, routerProvider); if (nsxMode == NetworkOffering.NsxMode.NATTED) { serviceProviderMap.put(Service.SourceNat, Provider.Nsx); + serviceProviderMap.put(Service.StaticNat, Provider.Nsx); + serviceProviderMap.put(Service.PortForwarding, Provider.Nsx); + serviceProviderMap.put(Service.Lb, Provider.Nsx); } return serviceProviderMap; } From ff85d3df3b3be6707987eea724af7c6f1045b028 Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Mon, 30 Oct 2023 19:55:03 -0400 Subject: [PATCH 2/2] Nsx add lb rule (#8161) * NSX: Create and delete NSX Static Nat rules * fix issues with static nat * add static nat * Support to add and delete Port forward rules * add license * fix adding multiple pf rules * cleanup * NSX: Add support to create and delete Load balancer rules * fix deletion of lb rules * add header file and update protocol detail --- .../api/CreateNsxLoadBalancerRuleCommand.java | 63 +++++ .../api/DeleteNsxLoadBalancerRuleCommand.java | 40 ++++ .../agent/api/NsxNetworkCommand.java | 7 +- .../resource/NsxLoadBalancerMember.java | 41 ++++ .../cloudstack/resource/NsxNetworkRule.java | 34 +++ .../cloudstack/resource/NsxResource.java | 33 +++ .../cloudstack/service/NsxApiClient.java | 216 +++++++++++++++++- .../apache/cloudstack/service/NsxElement.java | 96 +++++++- .../cloudstack/service/NsxServiceImpl.java | 23 +- .../cloudstack/utils/NsxControllerUtils.java | 40 ++++ .../cloud/server/ConfigurationServerImpl.java | 1 + 11 files changed, 579 insertions(+), 15 deletions(-) create mode 100644 plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/agent/api/CreateNsxLoadBalancerRuleCommand.java create mode 100644 plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/agent/api/DeleteNsxLoadBalancerRuleCommand.java create mode 100644 plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/resource/NsxLoadBalancerMember.java diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/agent/api/CreateNsxLoadBalancerRuleCommand.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/agent/api/CreateNsxLoadBalancerRuleCommand.java new file mode 100644 index 00000000000..861fa34de50 --- /dev/null +++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/agent/api/CreateNsxLoadBalancerRuleCommand.java @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.agent.api; + +import org.apache.cloudstack.resource.NsxLoadBalancerMember; + +import java.util.List; + +public class CreateNsxLoadBalancerRuleCommand extends NsxNetworkCommand { + + private final String publicPort; + private final String algorithm; + private final String protocol; + List memberList; + + private final long lbId; + public CreateNsxLoadBalancerRuleCommand(long domainId, long accountId, long zoneId, Long networkResourceId, + String networkResourceName, boolean isResourceVpc, + List memberList, long lbId, String publicPort, + String algorithm, String protocol) { + super(domainId, accountId, zoneId, networkResourceId, networkResourceName, isResourceVpc); + this.lbId = lbId; + this.memberList = memberList; + this.publicPort = publicPort; + this.algorithm = algorithm; + this.protocol = protocol; + } + + + public long getLbId() { + return lbId; + } + + public String getPublicPort() { + return publicPort; + } + + public List getMemberList() { + return memberList; + } + + public String getAlgorithm() { + return algorithm; + } + + public String getProtocol() { + return protocol; + } +} diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/agent/api/DeleteNsxLoadBalancerRuleCommand.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/agent/api/DeleteNsxLoadBalancerRuleCommand.java new file mode 100644 index 00000000000..21296aff991 --- /dev/null +++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/agent/api/DeleteNsxLoadBalancerRuleCommand.java @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.agent.api; + +import org.apache.cloudstack.resource.NsxLoadBalancerMember; + +import java.util.List; + +public class DeleteNsxLoadBalancerRuleCommand extends NsxNetworkCommand { + private long lbId; + List memberList; + + public DeleteNsxLoadBalancerRuleCommand(long domainId, long accountId, long zoneId, Long networkResourceId, + String networkResourceName, boolean isResourceVpc, + List memberList, long lbId, long vmId) { + super(domainId, accountId, zoneId, networkResourceId, networkResourceName, isResourceVpc, vmId); + this.lbId = lbId; + this.memberList = memberList; + } + + public long getLbId() { + return lbId; + } + + public List getMemberList() { return memberList; } +} diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/agent/api/NsxNetworkCommand.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/agent/api/NsxNetworkCommand.java index e5f49af9c17..4cad50db356 100644 --- a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/agent/api/NsxNetworkCommand.java +++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/agent/api/NsxNetworkCommand.java @@ -38,11 +38,16 @@ public class NsxNetworkCommand extends NsxCommand { } public NsxNetworkCommand(long domainId, long accountId, long zoneId, Long networkResourceId, String networkResourceName, - boolean isResourceVpc, Long vmId) { + boolean isResourceVpc) { super(domainId, accountId, zoneId); this.networkResourceId = networkResourceId; this.networkResourceName = networkResourceName; this.isResourceVpc = isResourceVpc; + } + + public NsxNetworkCommand(long domainId, long accountId, long zoneId, Long networkResourceId, String networkResourceName, + boolean isResourceVpc, Long vmId) { + this(domainId, accountId, zoneId, networkResourceId, networkResourceName, isResourceVpc); this.vmId = vmId; } diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/resource/NsxLoadBalancerMember.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/resource/NsxLoadBalancerMember.java new file mode 100644 index 00000000000..00960ddb78a --- /dev/null +++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/resource/NsxLoadBalancerMember.java @@ -0,0 +1,41 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.resource; + +public class NsxLoadBalancerMember { + private long vmId; + private String vmIp; + private int port; + + public NsxLoadBalancerMember(long vmId, String vmIp, int port) { + this.vmId = vmId; + this.vmIp = vmIp; + this.port = port; + } + + public long getVmId() { + return vmId; + } + + public String getVmIp() { + return vmIp; + } + + public int getPort() { + return port; + } +} diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/resource/NsxNetworkRule.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/resource/NsxNetworkRule.java index fc670429a19..5d830f1347a 100644 --- a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/resource/NsxNetworkRule.java +++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/resource/NsxNetworkRule.java @@ -16,6 +16,8 @@ // under the License. package org.apache.cloudstack.resource; +import java.util.List; + public class NsxNetworkRule { private long domainId; private long accountId; @@ -30,6 +32,8 @@ public class NsxNetworkRule { private String publicPort; private String privatePort; private String protocol; + private String algorithm; + private List memberList; public long getDomainId() { return domainId; @@ -135,6 +139,22 @@ public class NsxNetworkRule { this.protocol = protocol; } + public void setAlgorithm(String algorithm) { + this.algorithm = algorithm; + } + + public String getAlgorithm() { + return algorithm; + } + + public List getMemberList() { + return memberList; + } + + public void setMemberList(List memberList) { + this.memberList = memberList; + } + public static final class Builder { private long domainId; private long accountId; @@ -150,6 +170,8 @@ public class NsxNetworkRule { private String publicPort; private String privatePort; private String protocol; + private String algorithm; + private List memberList; public Builder() { } @@ -220,6 +242,16 @@ public class NsxNetworkRule { return this; } + public Builder setAlgorithm(String algorithm) { + this.algorithm = algorithm; + return this; + } + + public Builder setMemberList(List memberList) { + this.memberList = memberList; + return this; + } + public NsxNetworkRule build() { NsxNetworkRule rule = new NsxNetworkRule(); rule.setDomainId(this.domainId); @@ -235,6 +267,8 @@ public class NsxNetworkRule { rule.setPrivatePort(this.privatePort); rule.setProtocol(this.protocol); rule.setRuleId(this.ruleId); + rule.setAlgorithm(this.algorithm); + rule.setMemberList(this.memberList); return rule; } } diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/resource/NsxResource.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/resource/NsxResource.java index 2a375bcd483..d3cec1f5997 100644 --- a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/resource/NsxResource.java +++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/resource/NsxResource.java @@ -36,10 +36,12 @@ import com.vmware.nsx_policy.model.SiteListResult; import org.apache.cloudstack.NsxAnswer; import org.apache.cloudstack.StartupNsxCommand; import org.apache.cloudstack.agent.api.CreateNsxDhcpRelayConfigCommand; +import org.apache.cloudstack.agent.api.CreateNsxLoadBalancerRuleCommand; import org.apache.cloudstack.agent.api.CreateNsxPortForwardRuleCommand; import org.apache.cloudstack.agent.api.CreateNsxSegmentCommand; import org.apache.cloudstack.agent.api.CreateNsxStaticNatCommand; import org.apache.cloudstack.agent.api.CreateNsxTier1GatewayCommand; +import org.apache.cloudstack.agent.api.DeleteNsxLoadBalancerRuleCommand; import org.apache.cloudstack.agent.api.DeleteNsxSegmentCommand; import org.apache.cloudstack.agent.api.DeleteNsxNatRuleCommand; import org.apache.cloudstack.agent.api.DeleteNsxTier1GatewayCommand; @@ -114,6 +116,10 @@ public class NsxResource implements ServerResource { return executeRequest((DeleteNsxNatRuleCommand) cmd); } else if (cmd instanceof CreateNsxPortForwardRuleCommand) { return executeRequest((CreateNsxPortForwardRuleCommand) cmd); + } else if (cmd instanceof CreateNsxLoadBalancerRuleCommand) { + return executeRequest((CreateNsxLoadBalancerRuleCommand) cmd); + } else if (cmd instanceof DeleteNsxLoadBalancerRuleCommand) { + return executeRequest((DeleteNsxLoadBalancerRuleCommand) cmd); } else { return Answer.createUnsupportedCommandAnswer(cmd); } @@ -401,6 +407,33 @@ public class NsxResource implements ServerResource { return new NsxAnswer(cmd, true, null); } + private NsxAnswer executeRequest(CreateNsxLoadBalancerRuleCommand cmd) { + String tier1GatewayName = NsxControllerUtils.getTier1GatewayName(cmd.getDomainId(), cmd.getAccountId(), cmd.getZoneId(), + cmd.getNetworkResourceId(), cmd.isResourceVpc()); + String ruleName = NsxControllerUtils.getLoadBalancerRuleName(tier1GatewayName, cmd.getLbId()); + try { + nsxApiClient.createAndAddNsxLbVirtualServer(tier1GatewayName, cmd.getLbId(), cmd.getPublicIp(), cmd.getPublicPort(), + cmd.getMemberList(), cmd.getAlgorithm(), cmd.getProtocol()); + } catch (Exception e) { + LOGGER.error(String.format("Failed to add NSX load balancer rule %s for network: %s", ruleName, cmd.getNetworkResourceName())); + return new NsxAnswer(cmd, new CloudRuntimeException(e.getMessage())); + } + return new NsxAnswer(cmd, true, null); + } + + private NsxAnswer executeRequest(DeleteNsxLoadBalancerRuleCommand cmd) { + String tier1GatewayName = NsxControllerUtils.getTier1GatewayName(cmd.getDomainId(), cmd.getAccountId(), + cmd.getZoneId(), cmd.getNetworkResourceId(), cmd.isResourceVpc()); + String ruleName = NsxControllerUtils.getLoadBalancerRuleName(tier1GatewayName, cmd.getLbId()); + try { + nsxApiClient.deleteNsxLbResources(tier1GatewayName, cmd.getLbId(), cmd.getVmId()); + } catch (Exception e) { + LOGGER.error(String.format("Failed to add NSX load balancer rule %s for network: %s", ruleName, cmd.getNetworkResourceName())); + return new NsxAnswer(cmd, new CloudRuntimeException(e.getMessage())); + } + return new NsxAnswer(cmd, true, null); + } + @Override public boolean start() { return true; diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxApiClient.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxApiClient.java index 534030a6211..1da9a82b424 100644 --- a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxApiClient.java +++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxApiClient.java @@ -22,6 +22,10 @@ import com.cloud.utils.exception.CloudRuntimeException; import com.vmware.nsx.model.TransportZone; import com.vmware.nsx.model.TransportZoneListResult; import com.vmware.nsx_policy.infra.DhcpRelayConfigs; +import com.vmware.nsx_policy.infra.LbAppProfiles; +import com.vmware.nsx_policy.infra.LbPools; +import com.vmware.nsx_policy.infra.LbServices; +import com.vmware.nsx_policy.infra.LbVirtualServers; import com.vmware.nsx_policy.infra.Segments; import com.vmware.nsx_policy.infra.Services; import com.vmware.nsx_policy.infra.Sites; @@ -33,6 +37,16 @@ import com.vmware.nsx_policy.model.ApiError; import com.vmware.nsx_policy.model.DhcpRelayConfig; import com.vmware.nsx_policy.model.EnforcementPointListResult; import com.vmware.nsx_policy.model.L4PortSetServiceEntry; +<<<<<<< HEAD +import com.vmware.nsx_policy.model.LBAppProfileListResult; +import com.vmware.nsx_policy.model.LBPool; +import com.vmware.nsx_policy.model.LBPoolListResult; +import com.vmware.nsx_policy.model.LBPoolMember; +import com.vmware.nsx_policy.model.LBService; +import com.vmware.nsx_policy.model.LBVirtualServer; +import com.vmware.nsx_policy.model.LBVirtualServerListResult; +======= +>>>>>>> ce1659e8fcce61e0a6e7c75190af668f75172c87 import com.vmware.nsx_policy.model.LocaleServicesListResult; import com.vmware.nsx_policy.model.PolicyNatRule; import com.vmware.nsx_policy.model.Segment; @@ -41,6 +55,7 @@ import com.vmware.nsx_policy.model.ServiceListResult; import com.vmware.nsx_policy.model.SiteListResult; import com.vmware.nsx_policy.model.Tier1; import com.vmware.vapi.bindings.Service; +import com.vmware.vapi.bindings.Structure; import com.vmware.vapi.bindings.StubConfiguration; import com.vmware.vapi.cis.authn.SecurityContextFactory; import com.vmware.vapi.client.ApiClient; @@ -51,14 +66,31 @@ import com.vmware.vapi.internal.protocol.RestProtocol; import com.vmware.vapi.internal.protocol.client.rest.authn.BasicAuthenticationAppender; import com.vmware.vapi.protocol.HttpConfiguration; import com.vmware.vapi.std.errors.Error; +import org.apache.cloudstack.resource.NsxLoadBalancerMember; import org.apache.cloudstack.utils.NsxControllerUtils; import org.apache.commons.collections.CollectionUtils; import org.apache.log4j.Logger; import java.util.List; +<<<<<<< HEAD +import java.util.Locale; +import java.util.Objects; +import java.util.Optional; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static org.apache.cloudstack.utils.NsxControllerUtils.getServerPoolMemberName; +import static org.apache.cloudstack.utils.NsxControllerUtils.getServerPoolName; +import static org.apache.cloudstack.utils.NsxControllerUtils.getServiceName; +import static org.apache.cloudstack.utils.NsxControllerUtils.getVirtualServerName; +import static org.apache.cloudstack.utils.NsxControllerUtils.getServiceEntryName; +import static org.apache.cloudstack.utils.NsxControllerUtils.getLoadBalancerName; +import static org.apache.cloudstack.utils.NsxControllerUtils.getLoadBalancerAlgorithm; +======= import java.util.Objects; import java.util.function.Function; import java.util.stream.Collectors; +>>>>>>> ce1659e8fcce61e0a6e7c75190af668f75172c87 public class NsxApiClient { @@ -96,6 +128,22 @@ public class NsxApiClient { BYPASS } +<<<<<<< HEAD + public enum LBAlgorithm { + ROUND_ROBIN, + LEAST_CONNECTION, + IP_HASH + } + + private enum LBSize { + SMALL, + MEDIUM, + LARGE, + XLARGE + } + +======= +>>>>>>> ce1659e8fcce61e0a6e7c75190af668f75172c87 public enum RouteAdvertisementType { TIER1_STATIC_ROUTES, TIER1_CONNECTED, TIER1_NAT, TIER1_LB_VIP, TIER1_LB_SNAT, TIER1_DNS_FORWARDER_IP, TIER1_IPSEC_LOCAL_ENDPOINT } @@ -380,6 +428,166 @@ public class NsxApiClient { } } + public void createNsxLbServerPool(List memberList, String tier1GatewayName, String lbServerPoolName, String algorithm) { + for (NsxLoadBalancerMember member : memberList) { + try { + String serverPoolMemberName = getServerPoolMemberName(tier1GatewayName, member.getVmId()); + LbPools lbPools = (LbPools) nsxService.apply(LbPools.class); + LBPoolMember lbPoolMember = new LBPoolMember.Builder() + .setDisplayName(serverPoolMemberName) + .setIpAddress(member.getVmIp()) + .setPort(String.valueOf(member.getPort())) + .build(); + LBPool lbPool = new LBPool.Builder() + .setId(lbServerPoolName) + .setDisplayName(lbServerPoolName) + .setAlgorithm(getLoadBalancerAlgorithm(algorithm)) + .setMembers(List.of(lbPoolMember)) + .build(); + lbPools.patch(lbServerPoolName, lbPool); + } catch (Error error) { + ApiError ae = error.getData()._convertTo(ApiError.class); + String msg = String.format("Failed to create NSX LB server pool, due to: %s", ae.getErrorMessage()); + LOGGER.error(msg); + throw new CloudRuntimeException(msg); + } + } + } + + public void createNsxLoadBalancer(String tier1GatewayName, long lbId) { + try { + String lbName = getLoadBalancerName(tier1GatewayName); + LbServices lbServices = (LbServices) nsxService.apply(LbServices.class); + LBService lbService = getLbService(lbName); + if (Objects.nonNull(lbService)) { + return; + } + lbService = new LBService.Builder() + .setId(lbName) + .setDisplayName(lbName) + .setEnabled(true) + .setSize(LBSize.SMALL.name()) + .setConnectivityPath(TIER_1_GATEWAY_PATH_PREFIX + tier1GatewayName) + .build(); + lbServices.patch(lbName, lbService); + } catch (Error error) { + ApiError ae = error.getData()._convertTo(ApiError.class); + String msg = String.format("Failed to create NSX load balancer, due to: %s", ae.getErrorMessage()); + LOGGER.error(msg); + throw new CloudRuntimeException(msg); + } + } + + public void createAndAddNsxLbVirtualServer(String tier1GatewayName, long lbId, String publicIp, String publicPort, + List memberList, String algorithm, String protocol) { + try { + String lbServerPoolName = getServerPoolName(tier1GatewayName, lbId); + createNsxLbServerPool(memberList, tier1GatewayName, lbServerPoolName, algorithm); + createNsxLoadBalancer(tier1GatewayName, lbId); + + String lbVirtualServerName = getVirtualServerName(tier1GatewayName, lbId); + String lbServiceName = getLoadBalancerName(tier1GatewayName); + LbVirtualServers lbVirtualServers = (LbVirtualServers) nsxService.apply(LbVirtualServers.class); + LBVirtualServer lbVirtualServer = new LBVirtualServer.Builder() + .setId(lbVirtualServerName) + .setDisplayName(lbVirtualServerName) + .setApplicationProfilePath(getLbProfileForProtocol(protocol)) + .setIpAddress(publicIp) + .setLbServicePath(getLbPath(lbServiceName)) + .setPoolPath(getLbPoolPath(lbServerPoolName)) + .setPorts(List.of(publicPort)) + .build(); + lbVirtualServers.patch(lbVirtualServerName, lbVirtualServer); + } catch (Error error) { + ApiError ae = error.getData()._convertTo(ApiError.class); + String msg = String.format("Failed to create and add NSX virtual server to the Load Balancer, due to: %s", ae.getErrorMessage()); + LOGGER.error(msg); + throw new CloudRuntimeException(msg); + } + } + + public void deleteNsxLbResources(String tier1GatewayName, long lbId, long vmId) { + try { + // Delete associated Virtual servers + LbVirtualServers lbVirtualServers = (LbVirtualServers) nsxService.apply(LbVirtualServers.class); + String lbVirtualServerName = getVirtualServerName(tier1GatewayName, lbId); + lbVirtualServers.delete(lbVirtualServerName, false); + + // Delete LB pool + LbPools lbPools = (LbPools) nsxService.apply(LbPools.class); + String lbServerPoolName = getServerPoolName(tier1GatewayName, lbId); + lbPools.delete(lbServerPoolName, false); + + // Delete load balancer + LBVirtualServerListResult lbVsListResult = lbVirtualServers.list(null, null, null, null, null, null); + LBPoolListResult lbPoolListResult = lbPools.list(null, null, null, null, null, null); + if (CollectionUtils.isEmpty(lbVsListResult.getResults()) && CollectionUtils.isEmpty(lbPoolListResult.getResults())) { + String lbName = getLoadBalancerName(tier1GatewayName); + LbServices lbServices = (LbServices) nsxService.apply(LbServices.class); + lbServices.delete(lbName, true); + } + + } catch (Error error) { + ApiError ae = error.getData()._convertTo(ApiError.class); + String msg = String.format("Failed to delete NSX Load Balancer resources, due to: %s", ae.getErrorMessage()); + LOGGER.error(msg); + throw new CloudRuntimeException(msg); + } + } + + private String getLbPoolPath(String lbPoolName) { + try { + LbPools lbPools = (LbPools) nsxService.apply(LbPools.class); + LBPool lbPool = lbPools.get(lbPoolName); + return Objects.nonNull(lbPool) ? lbPool.getPath() : null; + } catch (Error error) { + ApiError ae = error.getData()._convertTo(ApiError.class); + String msg = String.format("Failed to get NSX LB server pool, due to: %s", ae.getErrorMessage()); + LOGGER.error(msg); + throw new CloudRuntimeException(msg); + } + } + private LBService getLbService(String lbName) { + try { + LbServices lbServices = (LbServices) nsxService.apply(LbServices.class); + LBService lbService = lbServices.get(lbName); + if (Objects.nonNull(lbService)) { + return lbService; + } + } catch (Exception e) { + return null; + } + return null; + } + + private String getLbPath(String lbServiceName) { + try { + LbServices lbServices = (LbServices) nsxService.apply(LbServices.class); + LBService lbService = lbServices.get(lbServiceName); + return Objects.nonNull(lbService) ? lbService.getPath() : null; + } catch (Error error) { + ApiError ae = error.getData()._convertTo(ApiError.class); + String msg = String.format("Failed to get NSX LB server pool, due to: %s", ae.getErrorMessage()); + LOGGER.error(msg); + throw new CloudRuntimeException(msg); + } + } + + private String getLbProfileForProtocol(String protocol) { + try { + LbAppProfiles lbAppProfiles = (LbAppProfiles) nsxService.apply(LbAppProfiles.class); + LBAppProfileListResult lbAppProfileListResults = lbAppProfiles.list(null, null, + null, null, null, null); + Optional appProfile = lbAppProfileListResults.getResults().stream().filter(profile -> profile._getDataValue().getField("path").toString().contains(protocol.toLowerCase(Locale.ROOT))).findFirst(); + return appProfile.map(structure -> structure._getDataValue().getField("path").toString()).orElse(null); + } catch (Error error) { + ApiError ae = error.getData()._convertTo(ApiError.class); + String msg = String.format("Failed to list NSX LB App profiles, due to: %s", ae.getErrorMessage()); + LOGGER.error(msg); + throw new CloudRuntimeException(msg); + } + } + public String getNsxInfraServices(String ruleName, String port, String protocol) { try { Services service = (Services) nsxService.apply(Services.class); @@ -456,12 +664,4 @@ public class NsxApiClient { } return null; } - - private String getServiceName(String ruleName, String port, String protocol) { - return ruleName + "-SVC-" + port + "-" +protocol; - } - - private String getServiceEntryName(String ruleName, String port, String protocol) { - return ruleName + "-SE-" + port + "-" + protocol; - } } diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxElement.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxElement.java index c8ac091485b..7f1fef855a1 100644 --- a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxElement.java +++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxElement.java @@ -24,6 +24,7 @@ import com.cloud.agent.api.AgentControlCommand; import com.cloud.agent.api.Answer; import com.cloud.agent.api.Command; import com.cloud.agent.api.StartupCommand; +import com.cloud.agent.api.to.LoadBalancerTO; import com.cloud.api.ApiDBUtils; import com.cloud.dc.DataCenterVO; import com.cloud.dc.dao.DataCenterDao; @@ -45,6 +46,8 @@ import com.cloud.network.PhysicalNetworkServiceProvider; import com.cloud.network.PublicIpAddress; import com.cloud.network.dao.IPAddressDao; import com.cloud.network.dao.IPAddressVO; +import com.cloud.network.dao.LoadBalancerVMMapDao; +import com.cloud.network.dao.LoadBalancerVMMapVO; import com.cloud.network.dao.NetworkDao; import com.cloud.network.dao.NetworkVO; import com.cloud.network.dao.PhysicalNetworkDao; @@ -52,9 +55,11 @@ import com.cloud.network.dao.PhysicalNetworkVO; import com.cloud.network.element.DhcpServiceProvider; import com.cloud.network.element.DnsServiceProvider; import com.cloud.network.element.IpDeployer; +import com.cloud.network.element.LoadBalancingServiceProvider; import com.cloud.network.element.PortForwardingServiceProvider; import com.cloud.network.element.StaticNatServiceProvider; import com.cloud.network.element.VpcProvider; +import com.cloud.network.lb.LoadBalancingRule; import com.cloud.network.rules.FirewallRule; import com.cloud.network.rules.PortForwardingRule; import com.cloud.network.rules.StaticNat; @@ -83,12 +88,14 @@ import com.cloud.vm.VirtualMachineProfile; import com.cloud.vm.dao.VMInstanceDao; import net.sf.ehcache.config.InvalidConfigurationException; import org.apache.cloudstack.StartupNsxCommand; +import org.apache.cloudstack.resource.NsxLoadBalancerMember; import org.apache.cloudstack.resource.NsxNetworkRule; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; import javax.inject.Inject; import javax.naming.ConfigurationException; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Locale; @@ -98,8 +105,10 @@ import java.util.Set; import java.util.function.LongFunction; @Component -public class NsxElement extends AdapterBase implements DhcpServiceProvider, DnsServiceProvider, VpcProvider, - StaticNatServiceProvider, IpDeployer, PortForwardingServiceProvider, ResourceStateAdapter, Listener { +public class NsxElement extends AdapterBase implements DhcpServiceProvider, DnsServiceProvider, VpcProvider, + StaticNatServiceProvider, IpDeployer, PortForwardingServiceProvider, + LoadBalancingServiceProvider, ResourceStateAdapter, Listener { + @Inject AccountManager accountMgr; @@ -125,6 +134,8 @@ public class NsxElement extends AdapterBase implements DhcpServiceProvider, DnsS VMInstanceDao vmInstanceDao; @Inject VpcDao vpcDao; + @Inject + LoadBalancerVMMapDao lbVmMapDao; private static final Logger LOGGER = Logger.getLogger(NsxElement.class); @@ -505,7 +516,6 @@ public class NsxElement extends AdapterBase implements DhcpServiceProvider, DnsS String privatePort = getPrivatePortRange(rule); - // TODO: add builder to reduce signature params ; should we pass port range? NsxNetworkRule networkRule = new NsxNetworkRule.Builder() .setDomainId(domainId) .setAccountId(accountId) @@ -522,9 +532,13 @@ public class NsxElement extends AdapterBase implements DhcpServiceProvider, DnsS .setProtocol(rule.getProtocol().toUpperCase(Locale.ROOT)) .build(); if (rule.getState() == FirewallRule.State.Add) { - return nsxService.createPortForwardRule(networkRule); + if (!nsxService.createPortForwardRule(networkRule)) { + return false; + } } else if (rule.getState() == FirewallRule.State.Revoke) { - return nsxService.deletePortForwardRule(networkRule); + if (!nsxService.deletePortForwardRule(networkRule)) { + return false; + } } } return true; @@ -558,4 +572,76 @@ public class NsxElement extends AdapterBase implements DhcpServiceProvider, DnsS String.valueOf(rule.getSourcePortStart()) : String.valueOf(rule.getSourcePortStart()).concat("-").concat(String.valueOf(rule.getSourcePortEnd())); } + + @Override + public boolean applyLBRules(Network network, List rules) throws ResourceUnavailableException { + for (LoadBalancingRule loadBalancingRule : rules) { + if (loadBalancingRule.getState() == FirewallRule.State.Active) { + continue; + } + IPAddressVO publicIp = ipAddressDao.findByIpAndDcId(network.getDataCenterId(), + loadBalancingRule.getSourceIp().addr()); + + Pair vpcOrNetwork = getVpcOrNetwork(network.getVpcId(), network.getId()); + VpcVO vpc = vpcOrNetwork.first(); + NetworkVO networkVO = vpcOrNetwork.second(); + Long networkResourceId = Objects.nonNull(vpc) ? vpc.getId() : networkVO.getId(); + String networkResourceName = Objects.nonNull(vpc) ? vpc.getName() : networkVO.getName(); + boolean isVpcResource = Objects.nonNull(vpc); + long domainId = Objects.nonNull(vpc) ? vpc.getDomainId() : networkVO.getDomainId(); + long accountId = Objects.nonNull(vpc) ? vpc.getAccountId() : networkVO.getAccountId(); + long zoneId = Objects.nonNull(vpc) ? vpc.getZoneId() : networkVO.getDataCenterId(); + List lbMembers = getLoadBalancerMembers(loadBalancingRule); + NsxNetworkRule networkRule = new NsxNetworkRule.Builder() + .setDomainId(domainId) + .setAccountId(accountId) + .setZoneId(zoneId) + .setNetworkResourceId(networkResourceId) + .setNetworkResourceName(networkResourceName) + .setVpcResource(isVpcResource) + .setMemberList(lbMembers) + .setPublicIp(publicIp.getAddress().addr()) + .setPublicPort(String.valueOf(loadBalancingRule.getSourcePortStart())) + .setRuleId(loadBalancingRule.getId()) + .setProtocol(loadBalancingRule.getProtocol().toUpperCase(Locale.ROOT)) + .setAlgorithm(loadBalancingRule.getAlgorithm()) + .build(); + if (loadBalancingRule.getState() == FirewallRule.State.Add) { + if (!nsxService.createLbRule(networkRule)) { + return false; + } + } else if (loadBalancingRule.getState() == FirewallRule.State.Revoke) { + if (!nsxService.deleteLbRule(networkRule)) { + return false; + } + } + } + return true; + } + + @Override + public boolean validateLBRule(Network network, LoadBalancingRule rule) { + return true; + } + + @Override + public List updateHealthChecks(Network network, List lbrules) { + return new ArrayList<>(); + } + + @Override + public boolean handlesOnlyRulesInTransitionState() { + return false; + } + + private List getLoadBalancerMembers(LoadBalancingRule lbRule) { + List lbVms = lbVmMapDao.listByLoadBalancerId(lbRule.getId(), false); + List lbMembers = new ArrayList<>(); + + for (LoadBalancerVMMapVO lbVm : lbVms) { + NsxLoadBalancerMember member = new NsxLoadBalancerMember(lbVm.getInstanceId(), lbVm.getInstanceIp(), lbRule.getDefaultPortStart()); + lbMembers.add(member); + } + return lbMembers; + } } diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxServiceImpl.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxServiceImpl.java index c2aef1404fb..6283f27f65f 100644 --- a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxServiceImpl.java +++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxServiceImpl.java @@ -23,9 +23,11 @@ import com.cloud.network.vpc.VpcVO; import com.cloud.network.vpc.dao.VpcDao; import com.cloud.utils.exception.CloudRuntimeException; import org.apache.cloudstack.NsxAnswer; +import org.apache.cloudstack.agent.api.CreateNsxLoadBalancerRuleCommand; import org.apache.cloudstack.agent.api.CreateNsxPortForwardRuleCommand; import org.apache.cloudstack.agent.api.CreateNsxStaticNatCommand; import org.apache.cloudstack.agent.api.CreateNsxTier1GatewayCommand; +import org.apache.cloudstack.agent.api.DeleteNsxLoadBalancerRuleCommand; import org.apache.cloudstack.agent.api.DeleteNsxSegmentCommand; import org.apache.cloudstack.agent.api.DeleteNsxNatRuleCommand; import org.apache.cloudstack.agent.api.DeleteNsxTier1GatewayCommand; @@ -119,9 +121,28 @@ public class NsxServiceImpl implements NsxService { public boolean deletePortForwardRule(NsxNetworkRule netRule) { DeleteNsxNatRuleCommand deleteCmd = new DeleteNsxNatRuleCommand(netRule.getDomainId(), netRule.getAccountId(), netRule.getZoneId(), netRule.getNetworkResourceId(), - netRule.getNetworkResourceName(), netRule.isVpcResource(), netRule.getVmId(), netRule.getRuleId(), netRule.getPrivatePort(), netRule.getPublicPort()); + netRule.getNetworkResourceName(), netRule.isVpcResource(), netRule.getVmId(), netRule.getRuleId(), netRule.getPrivatePort(), netRule.getProtocol()); deleteCmd.setService(Network.Service.PortForwarding); NsxAnswer result = nsxControllerUtils.sendNsxCommand(deleteCmd, netRule.getZoneId()); return result.getResult(); } + + public boolean createLbRule(NsxNetworkRule netRule) { + CreateNsxLoadBalancerRuleCommand command = new CreateNsxLoadBalancerRuleCommand(netRule.getDomainId(), + netRule.getAccountId(), netRule.getZoneId(), netRule.getNetworkResourceId(), + netRule.getNetworkResourceName(), netRule.isVpcResource(), netRule.getMemberList(), netRule.getRuleId(), + netRule.getPublicPort(), netRule.getAlgorithm(), netRule.getProtocol()); + command.setPublicIp(netRule.getPublicIp()); + NsxAnswer result = nsxControllerUtils.sendNsxCommand(command, netRule.getZoneId()); + return result.getResult(); + } + + public boolean deleteLbRule(NsxNetworkRule netRule) { + DeleteNsxLoadBalancerRuleCommand command = new DeleteNsxLoadBalancerRuleCommand(netRule.getDomainId(), + netRule.getAccountId(), netRule.getZoneId(), netRule.getNetworkResourceId(), + netRule.getNetworkResourceName(), netRule.isVpcResource(), netRule.getMemberList(), netRule.getRuleId(), + netRule.getVmId()); + NsxAnswer result = nsxControllerUtils.sendNsxCommand(command, netRule.getZoneId()); + return result.getResult(); + } } diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/utils/NsxControllerUtils.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/utils/NsxControllerUtils.java index 8c545476a6e..d0416c807b3 100644 --- a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/utils/NsxControllerUtils.java +++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/utils/NsxControllerUtils.java @@ -23,6 +23,7 @@ import com.cloud.network.dao.NsxProviderDao; import com.cloud.network.element.NsxProviderVO; import org.apache.cloudstack.NsxAnswer; import org.apache.cloudstack.agent.api.NsxCommand; +import org.apache.cloudstack.service.NsxApiClient; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -90,4 +91,43 @@ public class NsxControllerUtils { String suffix = "-PF"; return getTier1GatewayName(domainId, accountId, zoneId, networkResourceId, isVpcResource) + suffix + ruleId; } + + public static String getServiceName(String ruleName, String port, String protocol) { + return ruleName + "-SVC-" + port + "-" +protocol; + } + + public static String getServiceEntryName(String ruleName, String port, String protocol) { + return ruleName + "-SE-" + port + "-" + protocol; + } + + public static String getLoadBalancerName(String tier1GatewayName) { + return tier1GatewayName + "-LB"; + } + + public static String getLoadBalancerRuleName(String tier1GatewayName, long lbId) { + return tier1GatewayName + "-LB" + lbId; + } + + public static String getServerPoolName(String tier1GatewayName, long lbId) { + return getLoadBalancerRuleName(tier1GatewayName, lbId) + "-SP"; + } + + public static String getVirtualServerName(String tier1GatewayName, long lbId) { + return getLoadBalancerRuleName(tier1GatewayName, lbId) + "-VS"; + } + + public static String getServerPoolMemberName(String tier1GatewayName, long vmId) { + return tier1GatewayName + "-VM" + vmId; + } + + public static String getLoadBalancerAlgorithm(String algorithm) { + switch (algorithm) { + case "leastconn": + return NsxApiClient.LBAlgorithm.LEAST_CONNECTION.name(); + case "source": + return NsxApiClient.LBAlgorithm.IP_HASH.name(); + default: + return NsxApiClient.LBAlgorithm.ROUND_ROBIN.name(); + } + } } diff --git a/server/src/main/java/com/cloud/server/ConfigurationServerImpl.java b/server/src/main/java/com/cloud/server/ConfigurationServerImpl.java index 031f622e30d..69574b29ff7 100644 --- a/server/src/main/java/com/cloud/server/ConfigurationServerImpl.java +++ b/server/src/main/java/com/cloud/server/ConfigurationServerImpl.java @@ -1225,6 +1225,7 @@ public class ConfigurationServerImpl extends ManagerBase implements Configuratio new NetworkOfferingVO(name, displayText, TrafficType.Guest, false, false, null, null, true, Availability.Optional, null, GuestType.Isolated, false, false, false, false, false, forVpc); + defaultNatNSXNetworkOffering.setPublicLb(true); defaultNatNSXNetworkOffering.setForNsx(true); defaultNatNSXNetworkOffering.setNsxMode(nsxMode.name()); defaultNatNSXNetworkOffering.setState(NetworkOffering.State.Enabled);