diff --git a/scripts/vm/hypervisor/xenserver/cloudstack_pluginlib.py b/scripts/vm/hypervisor/xenserver/cloudstack_pluginlib.py index 422111f4f41..d2b95dc8930 100644 --- a/scripts/vm/hypervisor/xenserver/cloudstack_pluginlib.py +++ b/scripts/vm/hypervisor/xenserver/cloudstack_pluginlib.py @@ -21,6 +21,7 @@ import ConfigParser import logging import os import subprocess +import json from time import localtime, asctime @@ -176,6 +177,7 @@ def _build_flow_expr(**kwargs): dl_dst = 'dl_dst' in kwargs and ",dl_dst=%s" % kwargs['dl_dst'] or '' nw_src = 'nw_src' in kwargs and ",nw_src=%s" % kwargs['nw_src'] or '' nw_dst = 'nw_dst' in kwargs and ",nw_dst=%s" % kwargs['nw_dst'] or '' + table = 'table' in kwargs and ",table=%s" % kwargs['table'] or '' proto = 'proto' in kwargs and ",%s" % kwargs['proto'] or '' ip = ('nw_src' in kwargs or 'nw_dst' in kwargs) and ',ip' or '' flow = (flow + in_port + dl_type + dl_src + dl_dst + @@ -219,3 +221,155 @@ def del_all_flows(bridge): def del_port(bridge, port): delPort = [VSCTL_PATH, "del-port", bridge, port] do_cmd(delPort) + +def get_network_id_for_vif(vif_name): + domain_id, device_id = vif_name[3:len(vif_name)].split(".") + dom_uuid = do_cmd([XE_PATH, "vm-list", "dom-id=%s" % domain_id, "--minimal"]) + vif_uuid = do_cmd([XE_PATH, "vif-list", "vm-uuid=%s" % dom_uuid, "device=%s" % device_id, "--minimal"]) + vnet = do_cmd([XE_PATH, "vif-param-get", "uuid=%s" % vif_uuid, "param-name=other-config", + "param-key=cloudstack-network-id"]) + return vnet + +def get_network_id_for_tunnel_port(tunnelif_name): + vnet = do_cmd([VSCTL_PATH, "get", "interface", tunnelif_name, "options:cloudstack-network-id"]) + return vnet + +def clear_flooding_rules_for_port(bridge, ofport): + del_flows(bridge, in_port=ofport, table=2) + +def add_flooding_rules_for_port(bridge, in_ofport, out_ofports): + action = "".join("output:%s," %ofport for ofport in out_ofports)[:-1] + add_flow(bridge, priority=1100, in_port=in_ofport, table=1, actions=action) + +def get_ofport_for_vif(vif_name): + return do_cmd([VSCTL_PATH, "get", "interface", vif_name, "ofport"]) + +def get_macaddress_of_vif(vif_name): + domain_id, device_id = vif_name[3:len(vif_name)].split(".") + dom_uuid = do_cmd([XE_PATH, "vm-list", "dom-id=%s" % domain_id, "--minimal"]) + vif_uuid = do_cmd([XE_PATH, "vif-list", "vm-uuid=%s" % dom_uuid, "device=%s" % device_id, "--minimal"]) + mac = do_cmd([XE_PATH, "vif-param-get", "uuid=%s" % vif_uuid, "param-name=MAC"]) + return mac + +def get_vif_name_from_macaddress(macaddress): + vif_uuid = do_cmd([XE_PATH, "vif-list", "MAC=%s" % macaddress, "--minimal"]) + vif_device_id = do_cmd([XE_PATH, "vif-param-get", "uuid=%s" % vif_uuid, "param-name=device"]) + vm_uuid = do_cmd([XE_PATH, "vif-param-get", "uuid=%s" % vif_uuid, "param-name=vm-uuid"]) + vm_domain_id = do_cmd([XE_PATH, "vm-param-get", "uuid=%s" % vm_uuid, "param-name=dom-id"]) + return "vif"+vm_domain_id+"."+vif_device_id + +def add_mac_lookup_table_entry(bridge, mac_address, out_of_port): + add_flow(bridge, priority=1100, dl_dst=mac_address, table=1, actions="output:%s" % out_of_port) + +def delete_mac_lookup_table_entry(bridge, mac_address): + del_flows(bridge, dl_dst=mac_address, table=1) + +def add_ip_lookup_table_entry(bridge, ip, dst_tier_gateway_mac, dst_vm_mac): + action_str = "mod_dl_sr:%s" % dst_tier_gateway_mac + ",mod_dl_dst:%s" % dst_vm_mac +",resubmit(,5)" + addflow = [OFCTL_PATH, "add-flow", bridge, "table=4", "nw_dst=%s" % ip, "actions=%s" %action_str] + do_cmd(addflow) + +def get_vms_on_host(vpc, host_id): + all_vms = vpc.vms + vms_on_host = [] + for vm in all_vms: + if vm.hostid == host_id: + vms_on_host.append(vm) + return vms_on_host + +def get_network_details(vpc, network_uuid): + tiers = vpc.tiers + for tier in tiers: + if tier.networkuuid == network_uuid: + return tier + return None + +class jsonLoader(object): + def __init__(self, obj): + for k in obj: + v = obj[k] + if isinstance(v, dict): + setattr(self, k, jsonLoader(v)) + elif isinstance(v, (list, tuple)): + if len(v) > 0 and isinstance(v[0], dict): + setattr(self, k, [jsonLoader(elem) for elem in v]) + else: + setattr(self, k, v) + else: + setattr(self, k, v) + + def __getattr__(self, val): + if val in self.__dict__: + return self.__dict__[val] + else: + return None + + def __repr__(self): + return '{%s}' % str(', '.join('%s : %s' % (k, repr(v)) for (k, v) + in self.__dict__.iteritems())) + + def __str__(self): + return '{%s}' % str(', '.join('%s : %s' % (k, repr(v)) for (k, v) + in self.__dict__.iteritems())) + +def configure_bridge_for_topology(bridge, this_host_id, json_config): + vpconfig = jsonLoader(json.loads(json_config)).vpc + + if vpconfig is None: + logging.debug("WARNING:Can't find VPC info in json config file") + return "FAILURE:IMPROPER_JSON_CONFG_FILE" + + # get the list of Vm's in the VPC from the JSON config + this_host_vms = get_vms_on_host(vpconfig, this_host_id) + + for vm in this_host_vms: + for nic in vm.nics: + mac_addr = nic.macaddress + ip = nic.ipaddress + vif_name = get_vif_name_from_macaddress(mac_addr) + of_port = get_ofport_for_vif(vif_name) + network = get_network_details(vpconfig, nic.networkuuid) + + # Add flow rule in L2 look up table, if the destination mac = MAC of the nic send packet on the found OFPORT + add_mac_lookup_table_entry(bridge, mac_addr, of_port) + + # Add flow rule in L3 look up table: if the destination IP = VM's IP then modify the packet + # to set DST MAC = VM's MAC, SRC MAC=tier gateway MAC and send to egress table + add_ip_lookup_table_entry(bridge, ip, network.gatewaymac, mac_addr) + + # Add flow entry to send with intra tier traffic from the NIC to L2 lookup path) + addflow = [OFCTL_PATH, "add-flow", bridge, "table=0", "in_port=%s" % of_port, + "nw_dst=%s" %network.cidr, "actions=resubmit(,1)"] + do_cmd(addflow) + + #add flow entry to send inter-tier traffic from the NIC to egress ACL table(to L3 lookup path) + addflow = [OFCTL_PATH, "add-flow", bridge, "table=0", "in_port=%s" % of_port, + "dl_dst=%s" %network.gatewaymac, "nw_dst=%s" %vpconfig.cidr, "actions=resubmit(,3)"] + do_cmd(addflow) + + # get the list of hosts on which VPC spans from the JSON config + vpc_spanning_hosts = vpconfig.hosts + + for host in vpc_spanning_hosts: + if this_host_id == host.hostid: + continue + other_host_vms = get_vms_on_host(vpconfig, host.hostid) + for vm in other_host_vms: + for nic in vm.nics: + mac_addr = nic.macaddress + ip = nic.ipaddress + network = get_network_details(vpconfig, nic.networkuuid) + gre_key = network.grekey + + # generate tunnel name from tunnel naming convention + tunnel_name = "t%s-%s-%s" % (gre_key, this_host_id, host.hostid) + of_port = get_ofport_for_vif(tunnel_name) + + # Add flow rule in L2 look up table, if the destination mac = MAC of the nic send packet tunnel port + add_mac_lookup_table_entry(bridge, mac_addr, of_port) + + # Add flow tule in L3 look up table: if the destination IP = VM's IP then modify the packet + # set DST MAC = VM's MAC, SRC MAC=tier gateway MAC and send to egress table + add_ip_lookup_table_entry(bridge, ip, network.gatewaymac, mac_addr) + + return "SUCCESS: successfully configured bridge as per the VPC toplogy" \ No newline at end of file diff --git a/scripts/vm/hypervisor/xenserver/ovs-vif-flows.py b/scripts/vm/hypervisor/xenserver/ovs-vif-flows.py index 8452daef147..ae375252e22 100644 --- a/scripts/vm/hypervisor/xenserver/ovs-vif-flows.py +++ b/scripts/vm/hypervisor/xenserver/ovs-vif-flows.py @@ -18,6 +18,7 @@ # A simple script for enabling and disabling per-vif rules for explicitly # allowing broadcast/multicast traffic on the port where the VIF is attached +import copy import os import sys @@ -65,7 +66,6 @@ def clear_rules(vif): except: pass - def main(command, vif_raw): if command not in ('online', 'offline'): return @@ -86,39 +86,111 @@ def main(command, vif_raw): # find xs network for this bridge, verify is used for ovs tunnel network xs_nw_uuid = pluginlib.do_cmd([pluginlib.XE_PATH, "network-list", "bridge=%s" % bridge, "--minimal"]) - result = pluginlib.do_cmd([pluginlib.XE_PATH,"network-param-get", + ovs_tunnel_network = pluginlib.do_cmd([pluginlib.XE_PATH,"network-param-get", "uuid=%s" % xs_nw_uuid, "param-name=other-config", "param-key=is-ovs-tun-network", "--minimal"]) - if result != 'True': - return - - vlan = pluginlib.do_cmd([pluginlib.VSCTL_PATH, 'br-to-vlan', bridge]) - if vlan != '0': - # We need the REAL bridge name - bridge = pluginlib.do_cmd([pluginlib.VSCTL_PATH, - 'br-to-parent', bridge]) + ovs_vpc_distributed_vr_network = pluginlib.do_cmd([pluginlib.XE_PATH,"network-param-get", + "uuid=%s" % xs_nw_uuid, + "param-name=other-config", + "param-key=is-ovs_vpc_distributed_vr_network", "--minimal"]) - vsctl_output = pluginlib.do_cmd([pluginlib.VSCTL_PATH, - 'list-ports', bridge]) - vifs = vsctl_output.split('\n') - vif_ofports = [] - for vif in vifs: - vif_ofport = pluginlib.do_cmd([pluginlib.VSCTL_PATH, 'get', - 'Interface', vif, 'ofport']) - if this_vif == vif: - this_vif_ofport = vif_ofport - if vif.startswith('vif'): - vif_ofports.append(vif_ofport) + if ovs_tunnel_network == 'True': + vlan = pluginlib.do_cmd([pluginlib.VSCTL_PATH, 'br-to-vlan', bridge]) + if vlan != '0': + # We need the REAL bridge name + bridge = pluginlib.do_cmd([pluginlib.VSCTL_PATH, + 'br-to-parent', bridge]) + vsctl_output = pluginlib.do_cmd([pluginlib.VSCTL_PATH, + 'list-ports', bridge]) + vifs = vsctl_output.split('\n') + vif_ofports = [] + for vif in vifs: + vif_ofport = pluginlib.do_cmd([pluginlib.VSCTL_PATH, 'get', + 'Interface', vif, 'ofport']) + if this_vif == vif: + this_vif_ofport = vif_ofport + if vif.startswith('vif'): + vif_ofports.append(vif_ofport) - if command == 'offline': - clear_flows(bridge, this_vif_ofport, vif_ofports) + if command == 'offline': + clear_flows(bridge, this_vif_ofport, vif_ofports) - if command == 'online': - apply_flows(bridge, this_vif_ofport, vif_ofports) + if command == 'online': + apply_flows(bridge, this_vif_ofport, vif_ofports) + if ovs_vpc_distributed_vr_network == 'True': + vlan = pluginlib.do_cmd([pluginlib.VSCTL_PATH, 'br-to-vlan', bridge]) + if vlan != '0': + # We need the REAL bridge name + bridge = pluginlib.do_cmd([pluginlib.VSCTL_PATH, + 'br-to-parent', bridge]) + vsctl_output = pluginlib.do_cmd([pluginlib.VSCTL_PATH, + 'list-ports', bridge]) + vif_network_id = pluginlib.get_network_id_for_vif(this_vif) + vnet_vif_ofports = [] + vnet_tunnelif_ofports = [] + vnet_all_ofports = [] + + ports = vsctl_output.split('\n') + for port in ports: + if_ofport = pluginlib.do_cmd([pluginlib.VSCTL_PATH, 'get', 'Interface', vif, 'ofport']) + if vif.startswith('vif'): + # check VIF is in same network as that of plugged vif + if vif_network_id != pluginlib.get_network_id_for_vif(port): + continue + vnet_vif_ofports.append(if_ofport) + vnet_all_ofports.append(if_ofport) + + if vif.startswith('t'): + # check tunnel port is in same network as that of plugged vif + if vif_network_id != pluginlib.get_network_id_for_tunnel_port(port): + continue + vnet_tunnelif_ofports.append(if_ofport) + vnet_all_ofports.append(if_ofport) + + if command == 'online': + for port in vnet_all_ofports: + pluginlib.clear_flooding_rules_for_port(bridge, port) + + # for a packet arrived from tunnel port, flood only on VIF ports + for port in vnet_tunnelif_ofports: + pluginlib.add_flooding_rules_for_port(bridge, port, vnet_vif_ofports) + + # send on all VIF and tunnel port excluding the port on which packet arrived + for port in vnet_vif_ofports: + vnet_all_ofports_copy = copy.copy(vnet_all_ofports) + vnet_all_ofports_copy.remove(port) + pluginlib.add_flooding_rules_for_port(bridge, port, vnet_all_ofports_copy) + + #learn that MAC is reachable through the VIF port + mac = pluginlib.get_macaddress_of_vif(this_vif) + pluginlib.add_mac_lookup_table_entry(bridge, mac, this_vif_ofport) + + if command == 'offline': + for port in vnet_all_ofports: + pluginlib.clear_flooding_rules_for_port(bridge, port) + vnet_all_ofports.remove(this_vif_ofport) + vnet_vif_ofports.remove(this_vif_ofport) + + # for a packet arrived from tunnel port, flood only on VIF ports + for port in vnet_tunnelif_ofports: + pluginlib.add_flooding_rules_for_port(bridge, port, vnet_vif_ofports) + + # for a packet from VIF port send on all VIF's and tunnel ports excluding the port on which packet arrived + for port in vnet_vif_ofports: + vnet_all_ofports_copy = copy.copy(vnet_all_ofports) + vnet_all_ofports_copy.remove(port) + pluginlib.add_flooding_rules_for_port(bridge, port, vnet_all_ofports_copy) + + #un-learn that MAC is reachable through the VIF port + mac = pluginlib.get_macaddress_of_vif(this_vif) + pluginlib.delete_mac_lookup_table_entry(bridge, mac) + + return + if __name__ == "__main__": if len(sys.argv) != 3: print "usage: %s [online|offline] vif-domid-idx" % \ diff --git a/scripts/vm/hypervisor/xenserver/ovstunnel b/scripts/vm/hypervisor/xenserver/ovstunnel index 106be0441ce..d558e972cda 100755 --- a/scripts/vm/hypervisor/xenserver/ovstunnel +++ b/scripts/vm/hypervisor/xenserver/ovstunnel @@ -124,6 +124,75 @@ def setup_ovs_bridge(session, args): logging.debug("Setup_ovs_bridge completed with result:%s" % result) return result +@echo +def setup_ovs_bridge_for_distributed_routing(session, args): + bridge = args.pop("bridge") + key = args.pop("key") + xs_nw_uuid = args.pop("xs_nw_uuid") + cs_host_id = args.pop("cs_host_id") + + res = lib.check_switch() + if res != "SUCCESS": + return "FAILURE:%s" % res + + logging.debug("About to manually create the bridge:%s" % bridge) + # create a bridge with the same name as the xapi network + res = lib.do_cmd([lib.VSCTL_PATH, "--", "--may-exist", "add-br", bridge, + "--", "set", "bridge", bridge]) + logging.debug("Bridge has been manually created:%s" % res) + # TODO: Make sure xs-network-uuid is set into external_ids + lib.do_cmd([lib.VSCTL_PATH, "set", "Bridge", bridge, + "external_ids:xs-network-uuid=%s" % xs_nw_uuid]) + + # Non empty result means something went wrong + if res: + result = "FAILURE:%s" % res + else: + # Verify the bridge actually exists, with the gre_key properly set + res = lib.do_cmd([lib.VSCTL_PATH, "get", "bridge", + bridge, "other_config:gre_key"]) + if key in res: + result = "SUCCESS:%s" % bridge + else: + result = "FAILURE:%s" % res + # Finally note in the xenapi network object that the network has + # been configured + xs_nw_uuid = lib.do_cmd([lib.XE_PATH, "network-list", + "bridge=%s" % bridge, "--minimal"]) + lib.do_cmd([lib.XE_PATH, "network-param-set", "uuid=%s" % xs_nw_uuid, + "other-config:is-ovs_vpc_distributed_vr_network=True"]) + conf_hosts = lib.do_cmd([lib.XE_PATH, "network-param-get", + "uuid=%s" % xs_nw_uuid, + "param-name=other-config", + "param-key=ovs-host-setup", "--minimal"]) + conf_hosts = cs_host_id + (conf_hosts and ',%s' % conf_hosts or '') + lib.do_cmd([lib.XE_PATH, "network-param-set", "uuid=%s" % xs_nw_uuid, + "other-config:ovs-host-setup=%s" % conf_hosts]) + + # add a default flow rule to send broadcast and multi-cast packets to L2 flooding table + lib.add_flow(bridge, priority=1000, dl_dst='ff:ff:ff:ff:ff:ff', table=0, actions='resubmit(,2)') + lib.add_flow(bridge, priority=1000, nw_dst='224.0.0.0/24', table=0, actions='resubmit(,2)') + + # add a default flow rule to send uni-cast traffic to L2 lookup table + lib.add_flow(bridge, priority=0, table=0, actions='resubmit(,1)') + + # add a default rule to send unknown mac address to L2 flooding table + lib.add_flow(bridge, priority=0, table=1, actions='resubmit(,2)') + + # add a default rule in L2 flood table to drop packet + lib.add_flow(bridge, priority=0, table=2, actions='drop') + + # add a default rule in egress table to forward packet to L3 lookup table + lib.add_flow(bridge, priority=0, table=3, actions='resubmit(,4)') + + # add a default rule in L3 lookup table to forward packet to L2 lookup table + lib.add_flow(bridge, priority=0, table=4, actions='resubmit(,1)') + + # add a default rule in egress table to forward packet to L3 lookup table + lib.add_flow(bridge, priority=0, table=5, actions='drop') + + logging.debug("Setup_ovs_bridge completed with result:%s" % result) + return result @echo def destroy_ovs_bridge(session, args): @@ -220,13 +289,36 @@ def create_tunnel(session, args): # Ensure no trailing LF if tun_ofport.endswith('\n'): tun_ofport = tun_ofport[:-1] - # add flow entryies for dropping broadcast coming in from gre tunnel - lib.add_flow(bridge, priority=1000, in_port=tun_ofport, + # find xs network for this bridge, verify is used for ovs tunnel network + xs_nw_uuid = lib.do_cmd([lib.XE_PATH, "network-list", + "bridge=%s" % bridge, "--minimal"]) + ovs_tunnel_network = lib.do_cmd([lib.XE_PATH,"network-param-get", + "uuid=%s" % xs_nw_uuid, + "param-name=other-config", + "param-key=is-ovs-tun-network", "--minimal"]) + ovs_vpc_distributed_vr_network = lib.do_cmd([lib.XE_PATH,"network-param-get", + "uuid=%s" % xs_nw_uuid, + "param-name=other-config", + "param-key=is-ovs_vpc_distributed_vr_network", "--minimal"]) + if ovs_tunnel_network == 'True': + # add flow entryies for dropping broadcast coming in from gre tunnel + lib.add_flow(bridge, priority=1000, in_port=tun_ofport, dl_dst='ff:ff:ff:ff:ff:ff', actions='drop') - lib.add_flow(bridge, priority=1000, in_port=tun_ofport, + lib.add_flow(bridge, priority=1000, in_port=tun_ofport, nw_dst='224.0.0.0/24', actions='drop') - drop_flow_setup = True - logging.debug("Broadcast drop rules added") + drop_flow_setup = True + logging.debug("Broadcast drop rules added") + + if ovs_vpc_distributed_vr_network == 'True': + # add flow rules for dropping broadcast coming in from tunnel ports + lib.add_flow(bridge, priority=1000, in_port=tun_ofport, table=0, + dl_dst='ff:ff:ff:ff:ff:ff', actions='drop') + lib.add_flow(bridge, priority=1000, in_port=tun_ofport, table=0, + nw_dst='224.0.0.0/24', actions='drop') + + # add flow rule to send the traffic from tunnel ports to L2 switching table only + lib.add_flow(bridge, priority=1000, in_port=tun_ofport, table=0, actions='resubmit(,1)') + return "SUCCESS:%s" % name except: logging.debug("An unexpected error occured. Rolling back") @@ -293,10 +385,20 @@ def getLabel(session, args): return label return False +@echo +def configure_ovs_bridge_for_network_topology(session, args): + bridge = args.pop("bridge") + json_config = args.pop("config") + this_host_id = args.pop("host-id") + + return lib.configure_bridge_for_topology(bridge, this_host_id, json_config) + if __name__ == "__main__": XenAPIPlugin.dispatch({"create_tunnel": create_tunnel, "destroy_tunnel": destroy_tunnel, "setup_ovs_bridge": setup_ovs_bridge, "destroy_ovs_bridge": destroy_ovs_bridge, "is_xcp": is_xcp, - "getLabel": getLabel}) + "getLabel": getLabel, + "setup_ovs_bridge_for_distributed_routing": setup_ovs_bridge_for_distributed_routing, + "configure_ovs_bridge_for_network_topology": configure_ovs_bridge_for_network_topology})