CLOUDSTACK-6668: OVS distributed routing: ensure bridge is deleted when

last VM from the VPC is deleted on a host

OVS distributed routing: ensure bridge is deleted when last VM from the
VPC is deleted on a host. This fix ensures that bridge is
destroyed.
This commit is contained in:
Murali Reddy 2014-05-14 16:38:17 +05:30
parent 35cd61c463
commit 63f6888588
7 changed files with 111 additions and 80 deletions

View File

@ -982,6 +982,7 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe
Network.Record rec = new Network.Record();
Set<Network> networks = Network.getByNameLabel(conn, nwName);
if (networks.size() == 0) {
rec.nameDescription = "tunnel network id# " + nwName;
rec.nameLabel = nwName;
@ -1056,11 +1057,11 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe
}
}
private synchronized void destroyTunnelNetwork(Connection conn, String bridgeName) {
private synchronized void destroyTunnelNetwork(Connection conn, Network nw, long hostId) {
try {
Network nw = findOrCreateTunnelNetwork(conn, bridgeName);
String bridge = nw.getBridge(conn);
String result = callHostPlugin(conn, "ovstunnel", "destroy_ovs_bridge", "bridge", bridge);
String result = callHostPlugin(conn, "ovstunnel", "destroy_ovs_bridge", "bridge", bridge,
"cs_host_id", ((Long)hostId).toString());
String[] res = result.split(":");
if (res.length != 2 || !res[0].equalsIgnoreCase("SUCCESS")) {
//TODO: Should make this error not fatal?
@ -1666,14 +1667,12 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe
return cmd;
}
private void cleanUpTmpDomVif(Connection conn, Network nw) {
List<VIF> vifs;
synchronized (_tmpDom0Vif) {
vifs = _tmpDom0Vif;
_tmpDom0Vif = new ArrayList<VIF>();
}
private void cleanUpTmpDomVif(Connection conn, Network nw) throws XenAPIException, XmlRpcException {
for (VIF v : vifs) {
Pair<VM, VM.Record> vm = getControlDomain(conn);
VM dom0 = vm.first();
Set<VIF> dom0Vifs = dom0.getVIFs(conn);
for (VIF v : dom0Vifs) {
String vifName = "unknown";
try {
VIF.Record vifr = v.getRecord(conn);
@ -5266,12 +5265,17 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe
}
private Answer execute(OvsDestroyBridgeCommand cmd) {
Connection conn = getConnection();
Network nw = findOrCreateTunnelNetwork(conn, cmd.getBridgeName());
cleanUpTmpDomVif(conn, nw);
destroyTunnelNetwork(conn, cmd.getBridgeName());
s_logger.debug("OVS Bridge destroyed");
return new Answer(cmd, true, null);
try {
Connection conn = getConnection();
Network nw = findOrCreateTunnelNetwork(conn, cmd.getBridgeName());
cleanUpTmpDomVif(conn, nw);
destroyTunnelNetwork(conn, nw, cmd.getHostId());
s_logger.debug("OVS Bridge destroyed");
return new Answer(cmd, true, null);
} catch (Exception e) {
s_logger.warn("caught execption when destroying ovs bridge", e);
return new Answer(cmd, false, e.getMessage());
}
}
private Answer execute(OvsDestroyTunnelCommand cmd) {

View File

@ -22,10 +22,12 @@ public class OvsDestroyBridgeCommand extends Command {
Long networkId;
String name;
Long hostId;
public OvsDestroyBridgeCommand(Long networkId, String name) {
public OvsDestroyBridgeCommand(Long networkId, String name, Long hostId) {
this.networkId = networkId;
this.name = name;
this.hostId = hostId;
}
public Long getNetworkId() {
@ -36,6 +38,10 @@ public class OvsDestroyBridgeCommand extends Command {
return name;
}
public Long getHostId() {
return hostId;
}
@Override
public boolean executeInSequence() {
return true;

View File

@ -55,7 +55,7 @@ public interface OvsNetworkTopologyGuru extends Manager {
/**
* get the list of all Vm id's in the network that are running on the host
*/
public List<Long> getActiveVmsInNetworkOnHost(long vpcId, long hostId);
public List<Long> getActiveVmsInNetworkOnHost(long vpcId, long hostId, boolean includeVr);
/**
* get the list of all Vpc id's in which, a VM has a nic in the network that is part of VPC

View File

@ -181,7 +181,7 @@ public class OvsNetworkTopologyGuruImpl extends ManagerBase implements OvsNetwor
Set<Long> vmIdsSet = new HashSet<>();
List<? extends Network> vpcNetworks = _vpcMgr.getVpcNetworks(vpcId);
for (Network network : vpcNetworks) {
List<Long> networkVmIds = getActiveVmsInNetworkOnHost(network.getId(), hostId);
List<Long> networkVmIds = getActiveVmsInNetworkOnHost(network.getId(), hostId, false);
if (networkVmIds != null && !networkVmIds.isEmpty()) {
vmIdsSet.addAll(networkVmIds);
}
@ -195,21 +195,19 @@ public class OvsNetworkTopologyGuruImpl extends ManagerBase implements OvsNetwor
* get the list of all Vm id's in the network that are running on the host
*/
@Override
public List<Long> getActiveVmsInNetworkOnHost(long networkId, long hostId) {
public List<Long> getActiveVmsInNetworkOnHost(long networkId, long hostId, boolean includeVr) {
List <Long> vmIds = new ArrayList<>();
List<UserVmVO> vms = _userVmDao.listByNetworkIdAndStates(networkId,
VirtualMachine.State.Running, VirtualMachine.State.Starting, VirtualMachine.State.Stopping, VirtualMachine.State.Unknown,
VirtualMachine.State.Migrating);
VirtualMachine.State.Running, VirtualMachine.State.Migrating);
// Find routers for the network
List<DomainRouterVO> routers = _routerDao.findByNetwork(networkId);
if (vms != null) {
for (UserVmVO vm : vms) {
if (vm.getHostId() == hostId)
vmIds.add(vm.getId());
}
}
if (routers.size() != 0) {
if (routers.size() != 0 && includeVr) {
for (DomainRouterVO router: routers) {
if (router.getHostId() == hostId)
vmIds.add(router.getId());

View File

@ -209,8 +209,8 @@ public class OvsTunnelManagerImpl extends ManagerBase implements OvsTunnelManage
private void handleCreateTunnelAnswer(Answer[] answers) {
OvsCreateTunnelAnswer r = (OvsCreateTunnelAnswer)answers[0];
String s =
String.format("(hostIP:%1$s, remoteIP:%2$s, bridge:%3$s," + "greKey:%4$s, portName:%5$s)", r.getFromIp(), r.getToIp(), r.getBridge(), r.getKey(),
r.getInPortName());
String.format("(hostIP:%1$s, remoteIP:%2$s, bridge:%3$s," + "greKey:%4$s, portName:%5$s)",
r.getFromIp(), r.getToIp(), r.getBridge(), r.getKey(), r.getInPortName());
Long from = r.getFrom();
Long to = r.getTo();
long networkId = r.getNetworkId();
@ -476,9 +476,13 @@ public class OvsTunnelManagerImpl extends ManagerBase implements OvsTunnelManage
if (nw.getVpcId() != null && isVpcEnabledForDistributedRouter(nw.getVpcId())) {
List<Long> vmIds = _ovsNetworkToplogyGuru.getActiveVmsInVpcOnHost(nw.getVpcId(), host.getId());
if (vmIds != null && !vmIds.isEmpty()) {
return;
}
// there are not active VM's on this host belonging to any of the tiers in the VPC, so remove
// the host from the tunnel mesh network and destroy the bridge
List<? extends Network> vpcNetworks = _vpcMgr.getVpcNetworks(nw.getVpcId());
try {
for (Network network: vpcNetworks) {
@ -499,7 +503,8 @@ public class OvsTunnelManagerImpl extends ManagerBase implements OvsTunnelManage
}
}
Command cmd = new OvsDestroyBridgeCommand(nw.getId(), generateBridgeNameForVpc(nw.getVpcId()));
Command cmd = new OvsDestroyBridgeCommand(nw.getId(), generateBridgeNameForVpc(nw.getVpcId()),
host.getId());
s_logger.debug("Destroying bridge for network " + nw.getId() + " on host:" + host.getId());
Answer ans = _agentMgr.send(host.getId(), cmd);
handleDestroyBridgeAnswer(ans, host.getId(), nw.getId());
@ -507,7 +512,7 @@ public class OvsTunnelManagerImpl extends ManagerBase implements OvsTunnelManage
}
} else {
List<Long> vmIds = _ovsNetworkToplogyGuru.getActiveVmsInNetworkOnHost(nw.getId(), host.getId());
List<Long> vmIds = _ovsNetworkToplogyGuru.getActiveVmsInNetworkOnHost(nw.getId(), host.getId(), true);
if (vmIds != null && !vmIds.isEmpty()) {
return;
}
@ -516,7 +521,7 @@ public class OvsTunnelManagerImpl extends ManagerBase implements OvsTunnelManage
* the tunnels for this network */
int key = getGreKey(nw);
String bridgeName = generateBridgeName(nw, key);
Command cmd = new OvsDestroyBridgeCommand(nw.getId(), bridgeName);
Command cmd = new OvsDestroyBridgeCommand(nw.getId(), bridgeName, host.getId());
s_logger.debug("Destroying bridge for network " + nw.getId() + " on host:" + host.getId());
Answer ans = _agentMgr.send(host.getId(), cmd);
handleDestroyBridgeAnswer(ans, host.getId(), nw.getId());
@ -558,44 +563,66 @@ public class OvsTunnelManagerImpl extends ManagerBase implements OvsTunnelManage
protected void checkAndCreateVpcTunnelNetworks(Host host, long vpcId) {
long hostId = host.getId();
String bridgeName=generateBridgeNameForVpc(vpcId);
List<Long> vmIds = _ovsNetworkToplogyGuru.getActiveVmsInVpcOnHost(vpcId, hostId);
if (vmIds == null || vmIds.isEmpty()) {
// since this is the first VM from the VPC being launched on the host, first setup the bridge
try {
Commands cmds = new Commands(new OvsSetupBridgeCommand(bridgeName, hostId, null));
s_logger.debug("Ask host " + hostId + " to create bridge for vpc " + vpcId + " and configure the "
+ " bridge for distributed routing.");
Answer[] answers = _agentMgr.send(hostId, cmds);
handleSetupBridgeAnswer(answers);
} catch (OperationTimedoutException | AgentUnavailableException e) {
s_logger.warn("Ovs Tunnel network created tunnel failed", e);
}
// now that bridge is setup, populate network acl's before the VM gets created
OvsVpcRoutingPolicyConfigCommand cmd = prepareVpcRoutingPolicyUpdate(vpcId);
cmd.setSequenceNumber(getNextRoutingPolicyUpdateSequenceNumber(vpcId));
if (!sendVpcRoutingPolicyChangeUpdate(cmd, hostId, bridgeName)) {
s_logger.debug("Failed to send VPC routing policy change update to host : " + hostId +
". But moving on with sending the updates to the rest of the hosts.");
}
}
List<? extends Network> vpcNetworks = _vpcMgr.getVpcNetworks(vpcId);
List<Long> vpcSpannedHostIds = _ovsNetworkToplogyGuru.getVpcSpannedHosts(vpcId);
String bridgeName=generateBridgeNameForVpc(vpcId);
boolean bridgeNotSetup = true;
for (Network vpcNetwork: vpcNetworks) {
if (vpcNetwork.getState() != Network.State.Implemented &&
vpcNetwork.getState() != Network.State.Implementing && vpcNetwork.getState() != Network.State.Setup)
continue;
int key = getGreKey(vpcNetwork);
List<Long> toHostIds = new ArrayList<Long>();
List<Long> fromHostIds = new ArrayList<Long>();
OvsTunnelNetworkVO tunnelRecord = null;
for (Long rh : vpcSpannedHostIds) {
if (rh == hostId) {
continue;
}
OvsTunnelNetworkVO ta = _tunnelNetworkDao.getByFromToNetwork(hostId, rh.longValue(), vpcNetwork.getId());
// Try and create the tunnel even if a previous attempt failed
if (ta == null || ta.getState().equals(OvsTunnel.State.Failed.name())) {
tunnelRecord = _tunnelNetworkDao.getByFromToNetwork(hostId, rh.longValue(), vpcNetwork.getId());
// Try and create the tunnel if does not exit or previous attempt failed
if (tunnelRecord == null || tunnelRecord.getState().equals(OvsTunnel.State.Failed.name())) {
s_logger.debug("Attempting to create tunnel from:" + hostId + " to:" + rh.longValue());
if (ta == null) {
if (tunnelRecord == null) {
createTunnelRecord(hostId, rh.longValue(), vpcNetwork.getId(), key);
}
if (!toHostIds.contains(rh)) {
toHostIds.add(rh);
}
}
ta = _tunnelNetworkDao.getByFromToNetwork(rh.longValue(),
hostId, vpcNetwork.getId());
// Try and create the tunnel even if a previous attempt failed
if (ta == null || ta.getState().equals(OvsTunnel.State.Failed.name())) {
s_logger.debug("Attempting to create tunnel from:" +
rh.longValue() + " to:" + hostId);
if (ta == null) {
createTunnelRecord(rh.longValue(), hostId,
vpcNetwork.getId(), key);
tunnelRecord = _tunnelNetworkDao.getByFromToNetwork(rh.longValue(), hostId, vpcNetwork.getId());
// Try and create the tunnel if does not exit or previous attempt failed
if (tunnelRecord == null || tunnelRecord.getState().equals(OvsTunnel.State.Failed.name())) {
s_logger.debug("Attempting to create tunnel from:" + rh.longValue() + " to:" + hostId);
if (tunnelRecord == null) {
createTunnelRecord(rh.longValue(), hostId, vpcNetwork.getId(), key);
}
if (!fromHostIds.contains(rh)) {
fromHostIds.add(rh);
@ -615,20 +642,16 @@ public class OvsTunnelManagerImpl extends ManagerBase implements OvsTunnelManage
String otherIp = getGreEndpointIP(rHost, vpcNetwork);
if (otherIp == null)
throw new GreTunnelException(
"Unable to retrieve the remote "
+ "endpoint for the GRE tunnel."
"Unable to retrieve the remote endpoint for the GRE tunnel."
+ "Failure is on host:" + rHost.getId());
Commands cmds = new Commands(
new OvsCreateTunnelCommand(otherIp, key,
Long.valueOf(hostId), i, vpcNetwork.getId(), myIp, bridgeName,
vpcNetwork.getUuid()));
Commands cmds = new Commands( new OvsCreateTunnelCommand(otherIp, key, Long.valueOf(hostId),
i, vpcNetwork.getId(), myIp, bridgeName, vpcNetwork.getUuid()));
s_logger.debug("Attempting to create tunnel from:" + hostId + " to:" + i + " for the network "
+ vpcNetwork.getId());
s_logger.debug("Ask host " + hostId
+ " to create gre tunnel to " + i);
Answer[] answers = _agentMgr.send(hostId, cmds);
handleCreateTunnelAnswer(answers);
bridgeNotSetup = false;
}
for (Long i : fromHostIds) {
@ -641,27 +664,12 @@ public class OvsTunnelManagerImpl extends ManagerBase implements OvsTunnelManage
+ hostId);
Answer[] answers = _agentMgr.send(i, cmds);
handleCreateTunnelAnswer(answers);
bridgeNotSetup = false;
}
} catch (GreTunnelException | OperationTimedoutException | AgentUnavailableException e) {
// I really thing we should do a better handling of these exceptions
s_logger.warn("Ovs Tunnel network created tunnel failed", e);
}
}
// If no tunnels have been configured, perform the bridge setup
// anyway. This will ensure VIF rules will be triggered
if (bridgeNotSetup) {
try {
Commands cmds = new Commands(new OvsSetupBridgeCommand(bridgeName, hostId, null));
s_logger.debug("Ask host " + hostId + " to configure bridge for vpc");
Answer[] answers = _agentMgr.send(hostId, cmds);
handleSetupBridgeAnswer(answers);
} catch (OperationTimedoutException | AgentUnavailableException e) {
// I really thing we should do a better handling of these exceptions
s_logger.warn("Ovs Tunnel network created tunnel failed", e);
}
}
}
@Override

View File

@ -345,8 +345,11 @@ def get_acl(vpcconfig, required_acl_id):
return None
def check_tunnel_exists(bridge, tunnel_name):
res = do_cmd([VSCTL_PATH, "port-to-br", tunnel_name])
return res == bridge
try:
res = do_cmd([VSCTL_PATH, "port-to-br", tunnel_name])
return res == bridge
except:
return False
def create_tunnel(bridge, remote_ip, gre_key, src_host, dst_host, network_uuid):
@ -460,7 +463,7 @@ def create_tunnel(bridge, remote_ip, gre_key, src_host, dst_host, network_uuid):
logging.debug("Successfully created tunnel from host %s" %src_host + " to host %s" %dst_host +
" with GRE key %s" %gre_key)
return "SUCCESS:%s creation succeeded" % name
return "SUCCESS:%s" % name
except:
logging.debug("An unexpected error occured. Rolling back")
if tunnel_setup:
@ -548,7 +551,8 @@ def configure_vpc_bridge_for_network_topology(bridge, this_host_id, json_config,
# check if tunnel exists already, if not create a tunnel from this host to remote host
if not check_tunnel_exists(bridge, tunnel_name):
create_tunnel(bridge, host.ipaddress, gre_key, this_host_id, host.hostid, network.networkuuid)
create_tunnel(bridge, str(host.ipaddress), str(gre_key), this_host_id,
host.hostid, network.networkuuid)
of_port = get_ofport_for_vif(tunnel_name)

View File

@ -96,16 +96,16 @@ def setup_ovs_bridge(session, args):
result = "SUCCESS:%s" % bridge
else:
result = "FAILURE:%s" % res
lib.do_cmd([lib.XE_PATH, "network-param-set", "uuid=%s" % xs_nw_uuid,
"other-config:is-ovs-tun-network=True"])
# Finally note in the xenapi network object that the network has
# been configured
xs_nw_uuid = lib.do_cmd([lib.XE_PATH, "network-list",
"bridge=%s" % bridge, "--minimal"])
lib.do_cmd([lib.XE_PATH, "network-param-set", "uuid=%s" % xs_nw_uuid,
"other-config:is-ovs-tun-network=True"])
conf_hosts = lib.do_cmd([lib.XE_PATH, "network-param-get",
"uuid=%s" % xs_nw_uuid,
"param-name=other-config",
"param-key=ovs-host-setup", "--minimal"])
"param-key=ovs-host-setup"])
conf_hosts = cs_host_id + (conf_hosts and ',%s' % conf_hosts or '')
lib.do_cmd([lib.XE_PATH, "network-param-set", "uuid=%s" % xs_nw_uuid,
"other-config:ovs-host-setup=%s" % conf_hosts])
@ -160,7 +160,7 @@ def setup_ovs_bridge_for_distributed_routing(session, args):
conf_hosts = lib.do_cmd([lib.XE_PATH, "network-param-get",
"uuid=%s" % xs_nw_uuid,
"param-name=other-config",
"param-key=ovs-host-setup", "--minimal"])
"param-key=ovs-host-setup"])
conf_hosts = cs_host_id + (conf_hosts and ',%s' % conf_hosts or '')
lib.do_cmd([lib.XE_PATH, "network-param-set", "uuid=%s" % xs_nw_uuid,
"other-config:ovs-host-setup=%s" % conf_hosts])
@ -204,6 +204,7 @@ def setup_ovs_bridge_for_distributed_routing(session, args):
@echo
def destroy_ovs_bridge(session, args):
bridge = args.pop("bridge")
this_host_id = args.pop("cs_host_id")
res = lib.check_switch()
if res != "SUCCESS":
return res
@ -213,11 +214,21 @@ def destroy_ovs_bridge(session, args):
result = "FAILURE:%s" % res
else:
# Note that the bridge has been removed on xapi network object
xs_nw_uuid = lib.do_cmd([xePath, "network-list",
xs_nw_uuid = lib.do_cmd([lib.XE_PATH, "network-list",
"bridge=%s" % bridge, "--minimal"])
#FIXME: WOW, this an error
#lib.do_cmd([xePath,"network-param-set", "uuid=%s" % xs_nw_uuid,
# "other-config:ovs-setup=False"])
conf_hosts = lib.do_cmd([lib.XE_PATH, "network-param-get",
"uuid=%s" % xs_nw_uuid,
"param-name=other-config",
"param-key=ovs-host-setup"])
new_conf_hosts = ""
hosts = conf_hosts.split(',')
for host in hosts:
if str(host) == str(this_host_id):
continue
new_conf_hosts = host + "," + new_conf_hosts
new_conf_hosts = new_conf_hosts[:-1]
lib.do_cmd([lib.XE_PATH, "network-param-set", "uuid=%s" % xs_nw_uuid,
"other-config:ovs-host-setup=%s" % new_conf_hosts])
result = "SUCCESS:%s" % bridge
logging.debug("Destroy_ovs_bridge completed with result:%s" % result)