Scripts that use ovs-vsctl and ovs-ofctl to setup a bridge for VPC in

distributed routing mode, and setup flows appropriatley

script to handle the VPC topology sent from management server in JSOn
format. From the JSON file, reqired configuration (tunnel setup and flow
rules setup) is setup on the bridge
This commit is contained in:
Murali Reddy 2014-03-10 11:58:37 +05:30
parent 32ac021043
commit 100df92455
3 changed files with 359 additions and 31 deletions

View File

@ -21,6 +21,7 @@ import ConfigParser
import logging
import os
import subprocess
import json
from time import localtime, asctime
@ -176,6 +177,7 @@ def _build_flow_expr(**kwargs):
dl_dst = 'dl_dst' in kwargs and ",dl_dst=%s" % kwargs['dl_dst'] or ''
nw_src = 'nw_src' in kwargs and ",nw_src=%s" % kwargs['nw_src'] or ''
nw_dst = 'nw_dst' in kwargs and ",nw_dst=%s" % kwargs['nw_dst'] or ''
table = 'table' in kwargs and ",table=%s" % kwargs['table'] or ''
proto = 'proto' in kwargs and ",%s" % kwargs['proto'] or ''
ip = ('nw_src' in kwargs or 'nw_dst' in kwargs) and ',ip' or ''
flow = (flow + in_port + dl_type + dl_src + dl_dst +
@ -219,3 +221,155 @@ def del_all_flows(bridge):
def del_port(bridge, port):
delPort = [VSCTL_PATH, "del-port", bridge, port]
do_cmd(delPort)
def get_network_id_for_vif(vif_name):
domain_id, device_id = vif_name[3:len(vif_name)].split(".")
dom_uuid = do_cmd([XE_PATH, "vm-list", "dom-id=%s" % domain_id, "--minimal"])
vif_uuid = do_cmd([XE_PATH, "vif-list", "vm-uuid=%s" % dom_uuid, "device=%s" % device_id, "--minimal"])
vnet = do_cmd([XE_PATH, "vif-param-get", "uuid=%s" % vif_uuid, "param-name=other-config",
"param-key=cloudstack-network-id"])
return vnet
def get_network_id_for_tunnel_port(tunnelif_name):
vnet = do_cmd([VSCTL_PATH, "get", "interface", tunnelif_name, "options:cloudstack-network-id"])
return vnet
def clear_flooding_rules_for_port(bridge, ofport):
del_flows(bridge, in_port=ofport, table=2)
def add_flooding_rules_for_port(bridge, in_ofport, out_ofports):
action = "".join("output:%s," %ofport for ofport in out_ofports)[:-1]
add_flow(bridge, priority=1100, in_port=in_ofport, table=1, actions=action)
def get_ofport_for_vif(vif_name):
return do_cmd([VSCTL_PATH, "get", "interface", vif_name, "ofport"])
def get_macaddress_of_vif(vif_name):
domain_id, device_id = vif_name[3:len(vif_name)].split(".")
dom_uuid = do_cmd([XE_PATH, "vm-list", "dom-id=%s" % domain_id, "--minimal"])
vif_uuid = do_cmd([XE_PATH, "vif-list", "vm-uuid=%s" % dom_uuid, "device=%s" % device_id, "--minimal"])
mac = do_cmd([XE_PATH, "vif-param-get", "uuid=%s" % vif_uuid, "param-name=MAC"])
return mac
def get_vif_name_from_macaddress(macaddress):
vif_uuid = do_cmd([XE_PATH, "vif-list", "MAC=%s" % macaddress, "--minimal"])
vif_device_id = do_cmd([XE_PATH, "vif-param-get", "uuid=%s" % vif_uuid, "param-name=device"])
vm_uuid = do_cmd([XE_PATH, "vif-param-get", "uuid=%s" % vif_uuid, "param-name=vm-uuid"])
vm_domain_id = do_cmd([XE_PATH, "vm-param-get", "uuid=%s" % vm_uuid, "param-name=dom-id"])
return "vif"+vm_domain_id+"."+vif_device_id
def add_mac_lookup_table_entry(bridge, mac_address, out_of_port):
add_flow(bridge, priority=1100, dl_dst=mac_address, table=1, actions="output:%s" % out_of_port)
def delete_mac_lookup_table_entry(bridge, mac_address):
del_flows(bridge, dl_dst=mac_address, table=1)
def add_ip_lookup_table_entry(bridge, ip, dst_tier_gateway_mac, dst_vm_mac):
action_str = "mod_dl_sr:%s" % dst_tier_gateway_mac + ",mod_dl_dst:%s" % dst_vm_mac +",resubmit(,5)"
addflow = [OFCTL_PATH, "add-flow", bridge, "table=4", "nw_dst=%s" % ip, "actions=%s" %action_str]
do_cmd(addflow)
def get_vms_on_host(vpc, host_id):
all_vms = vpc.vms
vms_on_host = []
for vm in all_vms:
if vm.hostid == host_id:
vms_on_host.append(vm)
return vms_on_host
def get_network_details(vpc, network_uuid):
tiers = vpc.tiers
for tier in tiers:
if tier.networkuuid == network_uuid:
return tier
return None
class jsonLoader(object):
def __init__(self, obj):
for k in obj:
v = obj[k]
if isinstance(v, dict):
setattr(self, k, jsonLoader(v))
elif isinstance(v, (list, tuple)):
if len(v) > 0 and isinstance(v[0], dict):
setattr(self, k, [jsonLoader(elem) for elem in v])
else:
setattr(self, k, v)
else:
setattr(self, k, v)
def __getattr__(self, val):
if val in self.__dict__:
return self.__dict__[val]
else:
return None
def __repr__(self):
return '{%s}' % str(', '.join('%s : %s' % (k, repr(v)) for (k, v)
in self.__dict__.iteritems()))
def __str__(self):
return '{%s}' % str(', '.join('%s : %s' % (k, repr(v)) for (k, v)
in self.__dict__.iteritems()))
def configure_bridge_for_topology(bridge, this_host_id, json_config):
vpconfig = jsonLoader(json.loads(json_config)).vpc
if vpconfig is None:
logging.debug("WARNING:Can't find VPC info in json config file")
return "FAILURE:IMPROPER_JSON_CONFG_FILE"
# get the list of Vm's in the VPC from the JSON config
this_host_vms = get_vms_on_host(vpconfig, this_host_id)
for vm in this_host_vms:
for nic in vm.nics:
mac_addr = nic.macaddress
ip = nic.ipaddress
vif_name = get_vif_name_from_macaddress(mac_addr)
of_port = get_ofport_for_vif(vif_name)
network = get_network_details(vpconfig, nic.networkuuid)
# Add flow rule in L2 look up table, if the destination mac = MAC of the nic send packet on the found OFPORT
add_mac_lookup_table_entry(bridge, mac_addr, of_port)
# Add flow rule in L3 look up table: if the destination IP = VM's IP then modify the packet
# to set DST MAC = VM's MAC, SRC MAC=tier gateway MAC and send to egress table
add_ip_lookup_table_entry(bridge, ip, network.gatewaymac, mac_addr)
# Add flow entry to send with intra tier traffic from the NIC to L2 lookup path)
addflow = [OFCTL_PATH, "add-flow", bridge, "table=0", "in_port=%s" % of_port,
"nw_dst=%s" %network.cidr, "actions=resubmit(,1)"]
do_cmd(addflow)
#add flow entry to send inter-tier traffic from the NIC to egress ACL table(to L3 lookup path)
addflow = [OFCTL_PATH, "add-flow", bridge, "table=0", "in_port=%s" % of_port,
"dl_dst=%s" %network.gatewaymac, "nw_dst=%s" %vpconfig.cidr, "actions=resubmit(,3)"]
do_cmd(addflow)
# get the list of hosts on which VPC spans from the JSON config
vpc_spanning_hosts = vpconfig.hosts
for host in vpc_spanning_hosts:
if this_host_id == host.hostid:
continue
other_host_vms = get_vms_on_host(vpconfig, host.hostid)
for vm in other_host_vms:
for nic in vm.nics:
mac_addr = nic.macaddress
ip = nic.ipaddress
network = get_network_details(vpconfig, nic.networkuuid)
gre_key = network.grekey
# generate tunnel name from tunnel naming convention
tunnel_name = "t%s-%s-%s" % (gre_key, this_host_id, host.hostid)
of_port = get_ofport_for_vif(tunnel_name)
# Add flow rule in L2 look up table, if the destination mac = MAC of the nic send packet tunnel port
add_mac_lookup_table_entry(bridge, mac_addr, of_port)
# Add flow tule in L3 look up table: if the destination IP = VM's IP then modify the packet
# set DST MAC = VM's MAC, SRC MAC=tier gateway MAC and send to egress table
add_ip_lookup_table_entry(bridge, ip, network.gatewaymac, mac_addr)
return "SUCCESS: successfully configured bridge as per the VPC toplogy"

View File

@ -18,6 +18,7 @@
# A simple script for enabling and disabling per-vif rules for explicitly
# allowing broadcast/multicast traffic on the port where the VIF is attached
import copy
import os
import sys
@ -65,7 +66,6 @@ def clear_rules(vif):
except:
pass
def main(command, vif_raw):
if command not in ('online', 'offline'):
return
@ -86,39 +86,111 @@ def main(command, vif_raw):
# find xs network for this bridge, verify is used for ovs tunnel network
xs_nw_uuid = pluginlib.do_cmd([pluginlib.XE_PATH, "network-list",
"bridge=%s" % bridge, "--minimal"])
result = pluginlib.do_cmd([pluginlib.XE_PATH,"network-param-get",
ovs_tunnel_network = pluginlib.do_cmd([pluginlib.XE_PATH,"network-param-get",
"uuid=%s" % xs_nw_uuid,
"param-name=other-config",
"param-key=is-ovs-tun-network", "--minimal"])
if result != 'True':
return
vlan = pluginlib.do_cmd([pluginlib.VSCTL_PATH, 'br-to-vlan', bridge])
if vlan != '0':
# We need the REAL bridge name
bridge = pluginlib.do_cmd([pluginlib.VSCTL_PATH,
'br-to-parent', bridge])
ovs_vpc_distributed_vr_network = pluginlib.do_cmd([pluginlib.XE_PATH,"network-param-get",
"uuid=%s" % xs_nw_uuid,
"param-name=other-config",
"param-key=is-ovs_vpc_distributed_vr_network", "--minimal"])
vsctl_output = pluginlib.do_cmd([pluginlib.VSCTL_PATH,
'list-ports', bridge])
vifs = vsctl_output.split('\n')
vif_ofports = []
for vif in vifs:
vif_ofport = pluginlib.do_cmd([pluginlib.VSCTL_PATH, 'get',
'Interface', vif, 'ofport'])
if this_vif == vif:
this_vif_ofport = vif_ofport
if vif.startswith('vif'):
vif_ofports.append(vif_ofport)
if ovs_tunnel_network == 'True':
vlan = pluginlib.do_cmd([pluginlib.VSCTL_PATH, 'br-to-vlan', bridge])
if vlan != '0':
# We need the REAL bridge name
bridge = pluginlib.do_cmd([pluginlib.VSCTL_PATH,
'br-to-parent', bridge])
vsctl_output = pluginlib.do_cmd([pluginlib.VSCTL_PATH,
'list-ports', bridge])
vifs = vsctl_output.split('\n')
vif_ofports = []
for vif in vifs:
vif_ofport = pluginlib.do_cmd([pluginlib.VSCTL_PATH, 'get',
'Interface', vif, 'ofport'])
if this_vif == vif:
this_vif_ofport = vif_ofport
if vif.startswith('vif'):
vif_ofports.append(vif_ofport)
if command == 'offline':
clear_flows(bridge, this_vif_ofport, vif_ofports)
if command == 'offline':
clear_flows(bridge, this_vif_ofport, vif_ofports)
if command == 'online':
apply_flows(bridge, this_vif_ofport, vif_ofports)
if command == 'online':
apply_flows(bridge, this_vif_ofport, vif_ofports)
if ovs_vpc_distributed_vr_network == 'True':
vlan = pluginlib.do_cmd([pluginlib.VSCTL_PATH, 'br-to-vlan', bridge])
if vlan != '0':
# We need the REAL bridge name
bridge = pluginlib.do_cmd([pluginlib.VSCTL_PATH,
'br-to-parent', bridge])
vsctl_output = pluginlib.do_cmd([pluginlib.VSCTL_PATH,
'list-ports', bridge])
vif_network_id = pluginlib.get_network_id_for_vif(this_vif)
vnet_vif_ofports = []
vnet_tunnelif_ofports = []
vnet_all_ofports = []
ports = vsctl_output.split('\n')
for port in ports:
if_ofport = pluginlib.do_cmd([pluginlib.VSCTL_PATH, 'get', 'Interface', vif, 'ofport'])
if vif.startswith('vif'):
# check VIF is in same network as that of plugged vif
if vif_network_id != pluginlib.get_network_id_for_vif(port):
continue
vnet_vif_ofports.append(if_ofport)
vnet_all_ofports.append(if_ofport)
if vif.startswith('t'):
# check tunnel port is in same network as that of plugged vif
if vif_network_id != pluginlib.get_network_id_for_tunnel_port(port):
continue
vnet_tunnelif_ofports.append(if_ofport)
vnet_all_ofports.append(if_ofport)
if command == 'online':
for port in vnet_all_ofports:
pluginlib.clear_flooding_rules_for_port(bridge, port)
# for a packet arrived from tunnel port, flood only on VIF ports
for port in vnet_tunnelif_ofports:
pluginlib.add_flooding_rules_for_port(bridge, port, vnet_vif_ofports)
# send on all VIF and tunnel port excluding the port on which packet arrived
for port in vnet_vif_ofports:
vnet_all_ofports_copy = copy.copy(vnet_all_ofports)
vnet_all_ofports_copy.remove(port)
pluginlib.add_flooding_rules_for_port(bridge, port, vnet_all_ofports_copy)
#learn that MAC is reachable through the VIF port
mac = pluginlib.get_macaddress_of_vif(this_vif)
pluginlib.add_mac_lookup_table_entry(bridge, mac, this_vif_ofport)
if command == 'offline':
for port in vnet_all_ofports:
pluginlib.clear_flooding_rules_for_port(bridge, port)
vnet_all_ofports.remove(this_vif_ofport)
vnet_vif_ofports.remove(this_vif_ofport)
# for a packet arrived from tunnel port, flood only on VIF ports
for port in vnet_tunnelif_ofports:
pluginlib.add_flooding_rules_for_port(bridge, port, vnet_vif_ofports)
# for a packet from VIF port send on all VIF's and tunnel ports excluding the port on which packet arrived
for port in vnet_vif_ofports:
vnet_all_ofports_copy = copy.copy(vnet_all_ofports)
vnet_all_ofports_copy.remove(port)
pluginlib.add_flooding_rules_for_port(bridge, port, vnet_all_ofports_copy)
#un-learn that MAC is reachable through the VIF port
mac = pluginlib.get_macaddress_of_vif(this_vif)
pluginlib.delete_mac_lookup_table_entry(bridge, mac)
return
if __name__ == "__main__":
if len(sys.argv) != 3:
print "usage: %s [online|offline] vif-domid-idx" % \

View File

@ -124,6 +124,75 @@ def setup_ovs_bridge(session, args):
logging.debug("Setup_ovs_bridge completed with result:%s" % result)
return result
@echo
def setup_ovs_bridge_for_distributed_routing(session, args):
bridge = args.pop("bridge")
key = args.pop("key")
xs_nw_uuid = args.pop("xs_nw_uuid")
cs_host_id = args.pop("cs_host_id")
res = lib.check_switch()
if res != "SUCCESS":
return "FAILURE:%s" % res
logging.debug("About to manually create the bridge:%s" % bridge)
# create a bridge with the same name as the xapi network
res = lib.do_cmd([lib.VSCTL_PATH, "--", "--may-exist", "add-br", bridge,
"--", "set", "bridge", bridge])
logging.debug("Bridge has been manually created:%s" % res)
# TODO: Make sure xs-network-uuid is set into external_ids
lib.do_cmd([lib.VSCTL_PATH, "set", "Bridge", bridge,
"external_ids:xs-network-uuid=%s" % xs_nw_uuid])
# Non empty result means something went wrong
if res:
result = "FAILURE:%s" % res
else:
# Verify the bridge actually exists, with the gre_key properly set
res = lib.do_cmd([lib.VSCTL_PATH, "get", "bridge",
bridge, "other_config:gre_key"])
if key in res:
result = "SUCCESS:%s" % bridge
else:
result = "FAILURE:%s" % res
# Finally note in the xenapi network object that the network has
# been configured
xs_nw_uuid = lib.do_cmd([lib.XE_PATH, "network-list",
"bridge=%s" % bridge, "--minimal"])
lib.do_cmd([lib.XE_PATH, "network-param-set", "uuid=%s" % xs_nw_uuid,
"other-config:is-ovs_vpc_distributed_vr_network=True"])
conf_hosts = lib.do_cmd([lib.XE_PATH, "network-param-get",
"uuid=%s" % xs_nw_uuid,
"param-name=other-config",
"param-key=ovs-host-setup", "--minimal"])
conf_hosts = cs_host_id + (conf_hosts and ',%s' % conf_hosts or '')
lib.do_cmd([lib.XE_PATH, "network-param-set", "uuid=%s" % xs_nw_uuid,
"other-config:ovs-host-setup=%s" % conf_hosts])
# add a default flow rule to send broadcast and multi-cast packets to L2 flooding table
lib.add_flow(bridge, priority=1000, dl_dst='ff:ff:ff:ff:ff:ff', table=0, actions='resubmit(,2)')
lib.add_flow(bridge, priority=1000, nw_dst='224.0.0.0/24', table=0, actions='resubmit(,2)')
# add a default flow rule to send uni-cast traffic to L2 lookup table
lib.add_flow(bridge, priority=0, table=0, actions='resubmit(,1)')
# add a default rule to send unknown mac address to L2 flooding table
lib.add_flow(bridge, priority=0, table=1, actions='resubmit(,2)')
# add a default rule in L2 flood table to drop packet
lib.add_flow(bridge, priority=0, table=2, actions='drop')
# add a default rule in egress table to forward packet to L3 lookup table
lib.add_flow(bridge, priority=0, table=3, actions='resubmit(,4)')
# add a default rule in L3 lookup table to forward packet to L2 lookup table
lib.add_flow(bridge, priority=0, table=4, actions='resubmit(,1)')
# add a default rule in egress table to forward packet to L3 lookup table
lib.add_flow(bridge, priority=0, table=5, actions='drop')
logging.debug("Setup_ovs_bridge completed with result:%s" % result)
return result
@echo
def destroy_ovs_bridge(session, args):
@ -220,13 +289,36 @@ def create_tunnel(session, args):
# Ensure no trailing LF
if tun_ofport.endswith('\n'):
tun_ofport = tun_ofport[:-1]
# add flow entryies for dropping broadcast coming in from gre tunnel
lib.add_flow(bridge, priority=1000, in_port=tun_ofport,
# find xs network for this bridge, verify is used for ovs tunnel network
xs_nw_uuid = lib.do_cmd([lib.XE_PATH, "network-list",
"bridge=%s" % bridge, "--minimal"])
ovs_tunnel_network = lib.do_cmd([lib.XE_PATH,"network-param-get",
"uuid=%s" % xs_nw_uuid,
"param-name=other-config",
"param-key=is-ovs-tun-network", "--minimal"])
ovs_vpc_distributed_vr_network = lib.do_cmd([lib.XE_PATH,"network-param-get",
"uuid=%s" % xs_nw_uuid,
"param-name=other-config",
"param-key=is-ovs_vpc_distributed_vr_network", "--minimal"])
if ovs_tunnel_network == 'True':
# add flow entryies for dropping broadcast coming in from gre tunnel
lib.add_flow(bridge, priority=1000, in_port=tun_ofport,
dl_dst='ff:ff:ff:ff:ff:ff', actions='drop')
lib.add_flow(bridge, priority=1000, in_port=tun_ofport,
lib.add_flow(bridge, priority=1000, in_port=tun_ofport,
nw_dst='224.0.0.0/24', actions='drop')
drop_flow_setup = True
logging.debug("Broadcast drop rules added")
drop_flow_setup = True
logging.debug("Broadcast drop rules added")
if ovs_vpc_distributed_vr_network == 'True':
# add flow rules for dropping broadcast coming in from tunnel ports
lib.add_flow(bridge, priority=1000, in_port=tun_ofport, table=0,
dl_dst='ff:ff:ff:ff:ff:ff', actions='drop')
lib.add_flow(bridge, priority=1000, in_port=tun_ofport, table=0,
nw_dst='224.0.0.0/24', actions='drop')
# add flow rule to send the traffic from tunnel ports to L2 switching table only
lib.add_flow(bridge, priority=1000, in_port=tun_ofport, table=0, actions='resubmit(,1)')
return "SUCCESS:%s" % name
except:
logging.debug("An unexpected error occured. Rolling back")
@ -293,10 +385,20 @@ def getLabel(session, args):
return label
return False
@echo
def configure_ovs_bridge_for_network_topology(session, args):
bridge = args.pop("bridge")
json_config = args.pop("config")
this_host_id = args.pop("host-id")
return lib.configure_bridge_for_topology(bridge, this_host_id, json_config)
if __name__ == "__main__":
XenAPIPlugin.dispatch({"create_tunnel": create_tunnel,
"destroy_tunnel": destroy_tunnel,
"setup_ovs_bridge": setup_ovs_bridge,
"destroy_ovs_bridge": destroy_ovs_bridge,
"is_xcp": is_xcp,
"getLabel": getLabel})
"getLabel": getLabel,
"setup_ovs_bridge_for_distributed_routing": setup_ovs_bridge_for_distributed_routing,
"configure_ovs_bridge_for_network_topology": configure_ovs_bridge_for_network_topology})