From c6683c2eab5382188e6fa1abd4afb23962ad544c Mon Sep 17 00:00:00 2001 From: Frank Date: Tue, 11 Jan 2011 02:53:14 -0800 Subject: [PATCH] bug 7722: open vswitch - refine ovs plugin, create a separate plugin instead of messing with vmops refine gre tunnel, maintains tunnel in database instead of plugin fix an arp issue cause by overlap vlan range --- .../network/ovs/OvsCreateGreTunnelAnswer.java | 24 + .../ovs/OvsCreateGreTunnelCommand.java | 14 +- .../xen/resource/CitrixResourceBase.java | 23 +- scripts/vm/hypervisor/xenserver/ovsgre | 777 ++++++++++++++++++ .../vm/hypervisor/xenserver/vlanRemapUtils.py | 627 -------------- scripts/vm/hypervisor/xenserver/vmops | 284 +------ .../hypervisor/xenserver/xenserver56fp1/patch | 2 +- .../DefaultComponentLibrary.java | 2 + .../cloud/network/ovs/GreTunnelException.java | 7 + .../com/cloud/network/ovs/OvsListener.java | 72 +- .../cloud/network/ovs/OvsNetworkManager.java | 8 +- .../network/ovs/OvsNetworkManagerImpl.java | 170 ++-- .../cloud/network/ovs/dao/GreTunnelDao.java | 10 + .../network/ovs/dao/GreTunnelDaoImpl.java | 44 + .../cloud/network/ovs/dao/GreTunnelVO.java | 64 ++ .../cloud/network/ovs/dao/VlanMappingDao.java | 2 + .../network/ovs/dao/VlanMappingDaoImpl.java | 23 +- .../VirtualNetworkApplianceManagerImpl.java | 10 +- .../src/com/cloud/vm/UserVmManagerImpl.java | 11 +- setup/db/create-schema.sql | 8 + 20 files changed, 1160 insertions(+), 1022 deletions(-) create mode 100755 scripts/vm/hypervisor/xenserver/ovsgre delete mode 100644 scripts/vm/hypervisor/xenserver/vlanRemapUtils.py create mode 100644 server/src/com/cloud/network/ovs/GreTunnelException.java create mode 100644 server/src/com/cloud/network/ovs/dao/GreTunnelDao.java create mode 100644 server/src/com/cloud/network/ovs/dao/GreTunnelDaoImpl.java create mode 100644 server/src/com/cloud/network/ovs/dao/GreTunnelVO.java diff --git a/api/src/com/cloud/network/ovs/OvsCreateGreTunnelAnswer.java b/api/src/com/cloud/network/ovs/OvsCreateGreTunnelAnswer.java index ff299f1b0a5..bd541d9c1af 100644 --- a/api/src/com/cloud/network/ovs/OvsCreateGreTunnelAnswer.java +++ b/api/src/com/cloud/network/ovs/OvsCreateGreTunnelAnswer.java @@ -8,6 +8,9 @@ public class OvsCreateGreTunnelAnswer extends Answer { String remoteIp; String bridge; String key; + long from; + long to; + int port; public OvsCreateGreTunnelAnswer(Command cmd, boolean success, String details) { super(cmd, success, details); @@ -21,6 +24,15 @@ public class OvsCreateGreTunnelAnswer extends Answer { this.bridge = bridge; this.remoteIp = c.getRemoteIp(); this.key = c.getKey(); + this.port = -1; + this.from = c.getFrom(); + this.to = c.getTo(); + } + + public OvsCreateGreTunnelAnswer(Command cmd, boolean success, + String details, String hostIp, String bridge, int port) { + this(cmd, success, details, hostIp, bridge); + this.port = port; } public String getHostIp() { @@ -38,4 +50,16 @@ public class OvsCreateGreTunnelAnswer extends Answer { public String getKey() { return key; } + + public long getFrom() { + return from; + } + + public long getTo() { + return to; + } + + public int getPort() { + return port; + } } diff --git a/api/src/com/cloud/network/ovs/OvsCreateGreTunnelCommand.java b/api/src/com/cloud/network/ovs/OvsCreateGreTunnelCommand.java index 3cec9f69295..3c504a769df 100644 --- a/api/src/com/cloud/network/ovs/OvsCreateGreTunnelCommand.java +++ b/api/src/com/cloud/network/ovs/OvsCreateGreTunnelCommand.java @@ -5,15 +5,19 @@ import com.cloud.agent.api.Command; public class OvsCreateGreTunnelCommand extends Command { String remoteIp; String key; + long from; + long to; @Override public boolean executeInSequence() { return true; } - public OvsCreateGreTunnelCommand(String remoteIp, String key) { + public OvsCreateGreTunnelCommand(String remoteIp, String key, long from, long to) { this.remoteIp = remoteIp; this.key = key; + this.from = from; + this.to = to; } public String getRemoteIp() { @@ -23,4 +27,12 @@ public class OvsCreateGreTunnelCommand extends Command { public String getKey() { return key; } + + public long getFrom() { + return from; + } + + public long getTo() { + return to; + } } diff --git a/core/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java b/core/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java index b639b4884fe..5b15cfd2234 100644 --- a/core/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java +++ b/core/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java @@ -3835,8 +3835,8 @@ public abstract class CitrixResourceBase implements ServerResource { try { Network nw = setupvSwitchNetwork(conn); String bridge = nw.getBridge(conn); - String result = callHostPlugin(conn, "vmops", "ovs_delete_flow", "bridge", bridge, - "vmName", cmd.getVmName(), "reCreate", "true"); + String result = callHostPlugin(conn, "ovsgre", "ovs_delete_flow", "bridge", bridge, + "vmName", cmd.getVmName()); if (result.equalsIgnoreCase("SUCCESS")) { return new Answer(cmd, true, "success to delete flows for " + cmd.getVmName()); @@ -3852,7 +3852,7 @@ public abstract class CitrixResourceBase implements ServerResource { private List> ovsFullSyncStates() { Connection conn = getConnection(); try { - String result = callHostPlugin(conn, "vmops", "ovs_get_vm_log", "host_uuid", _host.uuid); + String result = callHostPlugin(conn, "ovsgre", "ovs_get_vm_log", "host_uuid", _host.uuid); String [] logs = result != null ?result.split(";"): new String [0]; List> states = new ArrayList>(); for (String log: logs){ @@ -3890,7 +3890,7 @@ public abstract class CitrixResourceBase implements ServerResource { * none guest network nic. don't worry, it will fail silently at host * plugin side */ - String result = callHostPlugin(conn, "vmops", "ovs_set_tag_and_flow", "bridge", bridge, + String result = callHostPlugin(conn, "ovsgre", "ovs_set_tag_and_flow", "bridge", bridge, "vmName", cmd.getVmName(), "tag", cmd.getTag(), "vlans", cmd.getVlans(), "seqno", cmd.getSeqNo()); s_logger.debug("set flow for " + cmd.getVmName() + " " + result); @@ -3917,15 +3917,16 @@ public abstract class CitrixResourceBase implements ServerResource { Network nw = setupvSwitchNetwork(conn); bridge = nw.getBridge(conn); - String result = callHostPlugin(conn, "vmops", "vlanRemapUtils", - "op", "createGRE", "bridge", bridge, - "remoteIP", cmd.getRemoteIp(), "greKey", cmd.getKey()); - if (result.equalsIgnoreCase("SUCCESS") || result.equalsIgnoreCase("TUNNEL_EXISTED")) { - return new OvsCreateGreTunnelAnswer(cmd, true, result, _host.ip, bridge); - } else { + String result = callHostPlugin(conn, "ovsgre", "ovs_create_gre", "bridge", bridge, + "remoteIP", cmd.getRemoteIp(), "greKey", cmd.getKey(), "from", + Long.toString(cmd.getFrom()), "to", Long.toString(cmd.getTo())); + String[] res = result.split(":"); + if (res.length != 2 || (res.length == 2 && res[1] == "[]")) { return new OvsCreateGreTunnelAnswer(cmd, false, result, _host.ip, bridge); - } + } else { + return new OvsCreateGreTunnelAnswer(cmd, true, result, _host.ip, bridge, Integer.parseInt(res[1])); + } } catch (Exception e) { e.printStackTrace(); } diff --git a/scripts/vm/hypervisor/xenserver/ovsgre b/scripts/vm/hypervisor/xenserver/ovsgre new file mode 100755 index 00000000000..99804ae63df --- /dev/null +++ b/scripts/vm/hypervisor/xenserver/ovsgre @@ -0,0 +1,777 @@ +#!/usr/bin/python +# +# A plugin for executing script needed by vmops cloud + +import os, sys, time +import XenAPIPlugin +sys.path.append("/opt/xensource/sm/") +import util +from util import CommandException +import hostvmstats +import socket +import stat +import base64 +import tempfile +from os.path import exists as _exists +from time import localtime as _localtime, asctime as _asctime + +vSwitchDBPidFile = "/var/run/openvswitch/ovsdb-server.pid" +vSwitchDBDaemonName = "ovsdb-server" +vSwitchPidFile = "/var/run/openvswitch/ovs-vswitchd.pid" +vsctlPath = "/usr/bin/ovs-vsctl" +vSwitchDaemonName = "ovs-vswitchd" + +logFile = "/var/log/ovsgre.log" +fLog = None + +global result + +errors = \ + {"NO_DB_PID_FILE" : "NO_DB_PID_FILE", \ + "DB_NOT_RUN" : "DB_NOT_RUN", \ + "NO_SWITCH_PID_FILE" : "NO_SWITCH_PID_FILE", \ + "SWITCH_NOT_RUN" : "SWITCH_NOT_RUN", \ + "NO_VSCTL" : "NO_VSCTL", \ + "COMMAND_FAILED" : "COMMAND_FAILED", \ + "TUNNEL_EXISTED" : "TUNNEL_EXISTED", \ + "NO_INPORT" : "NO_INPORT", \ + "NO_OFPORT" : "NO_OFPORT", \ + + "ERR_ARGS_NUM" : "ERR_ARGS_NUM", \ + "ERROR_OP" : "ERROR_OP", \ + "SUCCESS" : "SUCCESS", \ + } + +class ovs_log(object): + def __init__(self, name): + n = "ovs-%s" % name; + logfilename = "/var/run/cloud/" + n +".log" + + self.name = logfilename + self.vmName = name + self.bridge = "" + self.domId = "" + self.seqno = "" + self.tag = "" + self.vifs = "" + self.macs = "" + self.vlans = "" + self.ofports = "" + + def write(self): + log = open(self.name, "w") + log.write("vmName=%s" % self.vmName) + log.write("\n") + log.write("bridge=%s" % self.bridge) + log.write("\n") + log.write("domId=%s" % self.domId) + log.write("\n") + log.write("seqno=%s" % self.seqno) + log.write("\n") + log.write("tag=%s" % self.tag) + log.write("\n") + log.write("vifs=%s" % self.vifs) + log.write("\n") + log.write("macs=%s" % self.macs) + log.write("\n") + log.write("vlans=%s" % self.vlans) + log.write("\n") + log.write("ofports=%s" % self.ofports) + log.close() + + def read(self): + try: + lines = [line.rstrip() for line in open(self.name)] + for i in lines: + if "=" not in i: + util.SMlog("invalid line(%s) in %s" % (i, self.name)) + continue + + (key,value) = i.split("=") + if key == "vmName": + self.vmName = value + elif key == "bridge": + self.bridge = value + elif key == "domId": + self.domId = value + elif key == "seqno": + self.seqno = value + elif key == "tag": + self.tag = value + elif key == "vifs": + self.vifs = value + elif key == "macs": + self.macs = value + elif key == "vlans": + self.vlans = value + elif key == "ofports": + self.ofports = value + except Exception, e: + util.SMlog(e.__str__()) + util.SMlog("Failed to open ovs log %s" % self.name); + + def get_common_info(self): + self.read() + return "%s,%s,%s,%s,%s" % (self.vmName, self.bridge, self.domId, + self.seqno, self.tag) + + def remove(self): + try: + os.remove(self.name) + except: + util.SMlog("Failed to delete ovs log file " + self.name) + +def open_log (): + global fLog + + try: + if fLog == None: + fLog = open (logFile, "a") + except IOError, e: + #print e + pass + +def pr (str): + global fLog + + if fLog != None: + str = "[%s]:" % _asctime (_localtime()) + str + "\n" + fLog.write (str) + +def close_log (): + global fLog + + if fLog != None: + fLog.close () + +def is_process_run (pidFile, name): + try: + fpid = open (pidFile, "r") + pid = fpid.readline () + fpid.close () + except IOError, e: + return -1 + + pid = pid[:-1] + ps = os.popen ("ps -ae") + for l in ps: + if pid in l and name in l: + ps.close () + return 0 + + ps.close () + return -2 + +def is_tool_exist (name): + if _exists (name): + return 0 + return -1 + + +def check_switch (): + global result + + ret = is_process_run (vSwitchDBPidFile, vSwitchDBDaemonName); + if ret < 0: + if ret == -1: result = errors["NO_DB_PID_FILE"] + if ret == -2: result = errors["DB_NOT_RUN"] + return -1 + + ret = is_process_run (vSwitchPidFile, vSwitchDaemonName) + if ret < 0: + if ret == -1: result = errors["NO_SWITCH_PID_FILE"] + if ret == -2: result = errors["SWITCH_NOT_RUN"] + return -1 + + if is_tool_exist (vsctlPath) < 0: + result = errors["NO_VSCTL"] + return -1 + + return 0 + +def do_cmd (cmds, lines=False): + cmd = "" + for i in cmds: + cmd += " " + cmd += i + + pr("do command '%s'" % cmd) + f = os.popen (cmd) + if lines == True: + res = f.readlines () + else: + res = f.readline () + res = res[:-1] + f.close () + + if lines == False: + pr("command output '%s'" % res) + return res + +######################## GRE creation utils ########################## +# UUID's format is 8-4-4-4-12 +def is_uuid (uuid): + list = uuid.split ("-") + + if len (list) != 5: + return -1 + + if len (list[0]) != 8 or len (list[1]) != 4 \ + or len (list[2]) != 4 or len (list[3]) != 4 \ + or len (list[4]) != 12: + return -1 + + return 0 + +def check_gre (bridge, remoteIP, greKey): + ports = get_ports_on_bridge(bridge) + if ports == None: + return "[]" + + for i in ports: + ifaces = get_interface_on_port(i) + if ifaces == None: + continue + + for j in ifaces: + if j == '[]': + continue + options = get_field_of_interface(j, "options") + if remoteIP in options and greKey in options: + pr("WARNING: GRE tunnel for remote_ip=%s key=%s already here, \ +interface(%s)" % (remoteIP, greKey, j)) + return get_field_of_interface(j, "ofport") + + return "[]" + +def ovs_create_gre (session, args): + global result + bridge = args.pop("bridge") + remoteIP = args.pop("remoteIP") + greKey = args.pop("greKey") + srcHost = args.pop("from") + dstHost = args.pop("to") + + name = "%s-%s" % (srcHost, dstHost) + res = check_gre(bridge, remoteIP, greKey) + if res != "[]": + result = "SUCCESS:%s" % res + return result + + wait = [vsctlPath, "--timeout=30 wait-until bridge %s -- get bridge %s name" % \ + (bridge, bridge)] + res = do_cmd(wait) + if bridge not in res: + pr("WARNIING:Can't find bridge %s for creating tunnel!" % bridge) + result = errors["COMMAND_FAILED"] + return result + + createInterface = [vsctlPath, "create interface", "name=%s" % name, \ + 'type=gre options:"remote_ip=%s key=%s"' % (remoteIP, greKey)] + ifaceUUID = do_cmd (createInterface) + if is_uuid (ifaceUUID) < 0: + result = errors["COMMAND_FAILED"]; + return result + + createPort = [vsctlPath, "create port", "name=%s" % name, \ + "interfaces=[%s]" % ifaceUUID] + portUUID = do_cmd (createPort) + if is_uuid (portUUID) < 0: + result = errors["COMMAND_FAILED"]; + return result + + addBridge = [vsctlPath, "add bridge %s" % bridge, "ports %s" % portUUID] + do_cmd (addBridge) + + wait = [vsctlPath, "--timeout=5 wait-until port %s -- get port %s name" % \ + (name, name)] + res = do_cmd(wait) + if name in res: + port = get_field_of_interface(name, "ofport"); + result = "SUCCESS:%s" % port + else: + result = errors["COMMAND_FAILED"] + + return result +######################## End GRE creation utils ########################## + +######################## Flow creation utils ########################## +def get_ports_on_bridge(bridge): + listBr = [vsctlPath, "list br", bridge] + res = do_cmd(listBr, True) + + for i in res: + if "ports" in i: + (x, str) = i.split(":") + str = str.lstrip().rstrip() + str = str[1:] + str = str[:-1] + return str.split(",") + return None + +def get_filed_of_port(nameOruuid, field): + listport = [vsctlPath, "list port", nameOruuid] + res = do_cmd(listport, True) + + for i in res: + if field in i: + (x, r) = i.split(":") + return r.lstrip().rstrip() + return None + +def get_field_of_interface(nameOruuid, field): + listIface = [vsctlPath, "list interface", nameOruuid] + res = do_cmd(listIface, True) + + for i in res: + if field in i: + (x, r) = i.split(":") + return r.lstrip().rstrip() + return None + +def strip(str, direction="default"): + str = str.lstrip().rstrip() + if direction == "left": + return str[1:] + if direction == "right": + return str[:-1] + if direction == "both": + str = str[1:] + str = str[:-1] + return str + return str + +def get_vif_port(bridge, vifName): + portUuids = get_ports_on_bridge(bridge) + if portUuids == None: + pr("No ports on bridge %s" % bridge) + return None + + for i in portUuids: + name = get_filed_of_port(i, "name") + if name == None: + pr("WARNING: no name found for %s" % name) + continue + + name = strip(name, "both") + if name == vifName: + return get_field_of_interface(vifName, "ofport") + return None + +def get_interface_on_port(nameOruuid): + listPort = [vsctlPath, "list port", nameOruuid] + res = do_cmd(listPort, True) + + for i in res: + if "interfaces" in i: + (x, str) = i.split(":") + str = strip(str, "both") + return str.split(",") + return None + +def get_ofports_by_tag(bridge, tag): + portUuids = get_ports_on_bridge(bridge) + if portUuids == None: + pr("WARNING:No ports on bridge %s" % bridge) + return [] + + OfPorts = [] + for i in portUuids: + t = get_filed_of_port(i, "tag") + if t != tag: + pr("Skip port %s with tag=%s" % (i, t)) + continue + + iface = get_filed_of_port(i, "interfaces") + iface = strip(iface, "both") + port = get_field_of_interface(iface, "ofport") + if port != '[]': + OfPorts.append(port) + + return OfPorts + +def format_flow(inPort, vlan, mac, outPut): + flow = "in_port=%s dl_vlan=%s dl_dst=%s idle_timeout=0 hard_timeout=0 \ + priority=10000 actions=strip_vlan,output:%s" % (inPort, vlan, mac, outPut) + return flow + +def format_drop_flow(inPort, vlan): + flow = "in_port=%s dl_vlan=%s priority=0 idle_timeout=0 hard_timeout=0 \ + actions=drop" % (inPort, vlan) + return flow + +def del_flow(bridge, mac): + param = "dl_dst=%s" % mac + flow = ["ovs-ofctl del-flows %s" % bridge, '"%s"' % param] + do_cmd(flow) + +def del_arp_and_dhcp_flow(bridge, vlan, inPort): + param = "dl_type=0x0806 dl_vlan=%s in_port=%s" % (vlan, inPort) + flow = ["ovs-ofctl del-flows %s" % bridge, '"%s"' % param] + do_cmd(flow) + + param = "dl_type=0x0800 nw_proto=17 tp_dst=68 dl_vlan=%s, in_port=%s" % (vlan, inPort) + flow = ["ovs-ofctl del-flows %s" % bridge, '"%s"' % param] + do_cmd(flow) + + param = "dl_type=0x0800 nw_proto=17 tp_dst=67 dl_vlan=%s, in_port=%s" % (vlan, inPort) + flow = ["ovs-ofctl del-flows %s" % bridge, '"%s"' % param] + do_cmd(flow) + +def del_drop_flow(bridge, vlan): + param = "priority=0 dl_vlan=%s" % vlan + flow = ["ovs-ofctl del-flows %s" % bridge, '"%s"' % param] + do_cmd(flow) + +def format_normal_flow(): + flow = "priority=0 idle_timeout=0 hard_timeout=0 actions=normal" + return flow + +def format_dhcp_flow(bridge, inPort, vlan, ports): + outputs = '' + for i in ports: + str = "output:%s," % i + outputs += str + outputs = outputs[:-1] + flow = "in_port=%s dl_vlan=%s dl_type=0x0800 nw_proto=17 tp_dst=67 idle_timeout=0 hard_timeout=0 \ + priority=10000 actions=strip_vlan,%s" % (inPort, vlan, outputs) + return flow + +def format_dhcp_client_flow(bridge, inPort, vlan, ports): + outputs = '' + for i in ports: + str = "output:%s," % i + outputs += str + outputs = outputs[:-1] + flow = "in_port=%s dl_vlan=%s dl_type=0x0800 nw_proto=17 tp_dst=68 idle_timeout=0 hard_timeout=0 \ + priority=10000 actions=strip_vlan,%s" % (inPort, vlan, outputs) + return flow + +def format_arp_flow(bridge, inPort, vlan, ports): + outputs = '' + for i in ports: + str = "output:%s," % i + outputs += str + + outputs = outputs[:-1] + flow = "in_port=%s dl_vlan=%s dl_type=0x0806 idle_timeout=0 hard_timeout=0 \ + priority=10000 actions=strip_vlan,%s" % (inPort, vlan, outputs) + return flow + +def create_flow (bridge, vifName, mac, vlans): + global result + + output = get_vif_port(bridge, vifName) + if output == None: + pr("WARNING: cannot find ofport for %s" % vifName) + return errors["NO_OFPORT"] + return -1 + if output == '[]': + pr("WARNING: ofport is [] for %s" % vifName) + return errors["NO_OFPORT"] + return -1 + + #set remap here, remap has format e.g. 1/22/200/13/16 + pr("") + pr("Create flow for vlans=%s" % vlans) + for v in vlans.split(","): + try: + (vlan, inPort) = v.split(":") + flow = format_drop_flow(inPort, vlan) + add_flow(bridge, flow) + + flow = format_flow(inPort, vlan, mac, output) + add_flow(bridge, flow) + except Exception, e: + pr(e.__str__()) + pr("invalid map") + + # add normal flow make switch work as L2/L3 switch + flow = format_normal_flow() + add_flow(bridge, flow) + + return errors["SUCCESS"] +######################## End Flow creation utils ########################## + +def set_tag(bridge, vifName, vlan): + # The startVM command is slow, we may wait for a while for it creates vif on + # open vswitch + pr("Waiting for %s ..." % vifName) + waitPortCmd = [vsctlPath, "--timeout=10 wait-until port %s -- get port %s name" % \ + (vifName, vifName)] + do_cmd (waitPortCmd) + pr("%s is here" % vifName) + + if get_vif_port(bridge, vifName) == None: + pr("WARNING: %s is not on bridge %s" % (vifName, bridge)) + return 0 + + pr("Set tag") + set_tagCmd = [vsctlPath, "set port", vifName, "tag=%s"%vlan] + do_cmd (set_tagCmd) + return 0 + +def ask_ports(bridge, vifNames): + vifs = vifNames.split(",") + if len(vifs) == 0: + return ' ' + + ofports = [] + for vif in vifs: + op = get_vif_port(bridge, vif) + if op == None: + pr("ask_ports: no port(bridge:%s, vif:%s)" % (bridge, vif)) + continue + ofports.append(op) + + return ",".join(ofports) + +def delete_vm_flows(bridge, vmName, reCreate=True): + log = ovs_log(vmName) + log.read() + + macs = log.macs; + for i in macs.split(","): + del_flow(bridge, i) + pr("Delete flows for %s" % i) + + vlans = log.vlans + for v in vlans.split(","): + try: + (vlan, inPort) = v.split(":") + del_arp_and_dhcp_flow(bridge, vlan, inPort) + except Exception, e: + pr(e.__str__()) + pr("invalid map") + + for v in vlans: + try: + (vlan, inport) = v.split(":") + del_drop_flow(bridge, vlan) + except Exception, e: + pr(e.__str__()) + pr("invalid map") + + if reCreate == False: + return + + bridge = log.bridge + tag = log.tag + noneGreOfPorts = get_ofports_by_tag(bridge, tag) + + try: + noneGreOfPorts.remove(log.ofports) + except Exception, e: + pr(e.__str__()) + pr("ofport %s of %s is not on bridge %s" % (log.ofports, log.vmName, + bridge)) + + if len(noneGreOfPorts) != 0: + set_arp_and_dhcp_flow(bridge, vlans, tag, noneGreOfPorts) + + # add normal flow make switch work as L2/L3 switch + flow = format_normal_flow() + add_flow(bridge, flow) + + log.remove() + +def echo(fn): + def wrapped(*v, **k): + name = fn.__name__ + util.SMlog("#### VMOPS enter %s ####" % name ) + res = fn(*v, **k) + util.SMlog("#### VMOPS exit %s ####" % name ) + return res + return wrapped + + +def ovs_handle_rebooted_vm(session, vmName): + curr_domid = '-1' + log = ovs_log(vmName) + log.read() + + (curr_domid, vifrs, hostuuid) = ovs_get_domid_vifrs_hostuuid(session, vmName) + + old_id = log.domId; + if curr_domid == old_id: + util.SMlog("OvsInfo:%s is normal" % vmName) + return True + + util.SMlog("%s rebooted, reset flow for it" % vmName) + vlans = log.vlans; + bridge = log.bridge + tag = log.tag + for vifr in vifrs: + vifName = "vif" + curr_domid + "." + vifr[0] + set_tag(bridge, vifName, tag) + create_flow(bridge, vifName, vifr[1], vlans) + + log.domId = curr_domid + log.write() + + return True + +@echo +def ovs_get_vm_log(session, args): + host_uuid = args.pop('host_uuid') + try: + thishost = session.xenapi.host.get_by_uuid(host_uuid) + hostrec = session.xenapi.host.get_record(thishost) + vms = hostrec.get('resident_VMs') + except Exception, e: + util.SMlog("Failed to get host from uuid %s, exception: %s" % (host_uuid, e.__str__())) + return ' ' + + result = [] + try: + for name in [session.xenapi.VM.get_name_label(x) for x in vms]: + if 1 not in [ name.startswith(c) for c in ['r-', 'i-'] ]: + continue + ovs_handle_rebooted_vm(session, name) + if name.startswith('i-'): + log = ovs_log(name) + info = log.get_common_info() + result.append(info) + except Exception, e: + util.SMlog(e.__str__()) + util.SMlog("OVs failed to get logs, better luck next time!") + + return ";".join(result) + +def ovs_write_vm_log(bridge, vmName, vmId, seqno, vifNames, macs, tag, vlans, ofports): + log = ovs_log(vmName) + log.read() + log.bridge = bridge + log.vmName = vmName + log.domId = vmId + log.seqno = seqno + log.vifs = vifNames + log.macs = macs + log.tag = tag + log.vlans = vlans + log.ofports = ofports + log.write() + + util.SMlog("Writing ovs log to " + log.name) + +@echo +def ovs_delete_flow(session, args): + bridge = args.pop('bridge') + vmName = args.pop('vmName') + + delete_vm_flows(bridge, vmName) + return 'SUCCESS' + +def ovs_get_domid_vifrs_hostuuid(session, vmName): + def get_vif_field(name, field): + return session.xenapi.VIF.get_record(name).get(field) + + try: + vm = session.xenapi.VM.get_by_name_label(vmName) + if len(vm) != 1: + return 'NO_VM' + vm_rec = session.xenapi.VM.get_record(vm[0]) + vm_vifs = vm_rec.get('VIFs') + vifrs = [] + for vif in vm_vifs: + rec = (get_vif_field(vif, 'device'), get_vif_field(vif, 'MAC')) + vifrs.append(rec) + domid = vm_rec.get('domid') + host = vm_rec.get('resident_on') + host_rec = session.xenapi.host.get_record(host) + uuid = host_rec.get('uuid') + util.SMlog("OVSINFO: (domid:%s, uuid:%s)" % (domid, uuid)) + return (domid, vifrs, uuid) + + except: + util.SMlog("### Failed to get domid or vif list for vm ##" + vmName) + return (-1, [], "-1") + +def add_flow(bridge, flow): + param = bridge + ' "%s"' % flow + addflow = ["ovs-ofctl add-flow", param] + do_cmd (addflow) + +def set_arp_and_dhcp_flow(bridge, vlans, tag, ofports): + for v in vlans.split(","): + try: + (vlan, inPort) = v.split(":") + arpFlow = format_arp_flow(bridge, inPort, vlan, ofports) + add_flow(bridge, arpFlow) + + dhcpFlow = format_dhcp_flow(bridge, inPort, vlan, ofports) + add_flow(bridge, dhcpFlow) + + dhcpClientFlow = format_dhcp_client_flow(bridge, inPort, vlan, ofports) + add_flow(bridge, dhcpClientFlow) + except Exception, e: + pr(e.__str__()) + pr("invalid map") + +@echo +def ovs_set_arp_and_dhcp_flow(session, args): + vlans = args.pop('vlans') + bridge = args.pop('bridge') + tag = args.pop('tag') + + pr("ovs_set_arp_and_dhcp_flow: bridge=%s, vlans=%s, tag=%s" % (bridge, + vlans, tag)) + if vlans == '[]': + pr("No need to create arp and dhcp flow") + return 'SUCCESS' + + ofports = get_ofports_by_tag(bridge, tag) + if len(ofports) == 0: + pr("No VMs, skip set arp and dhcp flow for tag=%s" % tag) + return 'SUCCESS' + + set_arp_and_dhcp_flow(bridge, vlans, tag, ofports) + return 'SUCCESS' + +@echo +def ovs_set_tag_and_flow(session, args): + bridge = args.pop('bridge') + vmName = args.pop('vmName') + vlans = args.pop('vlans') + tag = args.pop('tag') + seqno = args.pop('seqno') + + (domid, vifrs, hostuuid) = ovs_get_domid_vifrs_hostuuid(session, vmName) + + if domid == '-1': + util.SMlog("### Failed to get domid for vm (-1): " + vmName) + return 'NO_DOMID' + + if len(vifrs) == 0: + return 'SUCCESS' + + pr("ovs_set_tag_and_flow: bridge=%s, vmName=%s, vlans=%s, tag=%s, seqno=%s" % + (bridge, vmName, vlans, tag, seqno)) + #delete old flows first + delete_vm_flows(bridge, vmName, False) + + vifNames = [] + vlans = strip(vlans, "both") + macs = [] + for vifr in vifrs: + vifName = "vif" + domid + "." + vifr[0] + vifNames.append(vifName) + mac = vifr[1] + macs.append(mac) + set_tag(bridge, vifName, tag) + create_flow(bridge, vifName, mac, vlans) + + vifs = ",".join(vifNames) + ofports = ask_ports(bridge, vifs) + ovs_write_vm_log(bridge, vmName, domid, seqno, vifs, ",".join(macs), tag, vlans, ofports) + + #see if there is rebooted vm to handle + ovs_get_vm_log(session, {"host_uuid":hostuuid}) + ovs_set_arp_and_dhcp_flow(session, {"bridge":bridge, "tag":tag, "vlans":vlans}) + return 'SUCCESS' + +if __name__ == "__main__": + open_log() + XenAPIPlugin.dispatch({"ovs_create_gre":ovs_create_gre, "ovs_set_tag_and_flow":ovs_set_tag_and_flow, "ovs_get_vm_log":ovs_get_vm_log,"ovs_delete_flow":ovs_delete_flow}) + close_log() + diff --git a/scripts/vm/hypervisor/xenserver/vlanRemapUtils.py b/scripts/vm/hypervisor/xenserver/vlanRemapUtils.py deleted file mode 100644 index 322d347007c..00000000000 --- a/scripts/vm/hypervisor/xenserver/vlanRemapUtils.py +++ /dev/null @@ -1,627 +0,0 @@ -#!/usr/bin/python - -# Create flows on Open vSwitch -# -# @param: bridge name, refer to a network created by xenserver -# @param: our MAC -# @param: vlan ID - -import os -import sys -from os.path import exists as _exists -from time import localtime as _localtime, asctime as _asctime - -vSwitchDBPidFile = "/var/run/openvswitch/ovsdb-server.pid" -vSwitchDBDaemonName = "ovsdb-server" -vSwitchPidFile = "/var/run/openvswitch/ovs-vswitchd.pid" -vsctlPath = "/usr/bin/ovs-vsctl" -vSwitchDaemonName = "ovs-vswitchd" - -logFile = "/var/log/vlanRemapUtils.log" -fLog = None - -global result - -errors = \ - {"NO_DB_PID_FILE" : "NO_DB_PID_FILE", \ - "DB_NOT_RUN" : "DB_NOT_RUN", \ - "NO_SWITCH_PID_FILE" : "NO_SWITCH_PID_FILE", \ - "SWITCH_NOT_RUN" : "SWITCH_NOT_RUN", \ - "NO_VSCTL" : "NO_VSCTL", \ - "COMMAND_FAILED" : "COMMAND_FAILED", \ - "TUNNEL_EXISTED" : "TUNNEL_EXISTED", \ - "NO_INPORT" : "NO_INPORT", \ - "NO_OFPORT" : "NO_OFPORT", \ - - "ERR_ARGS_NUM" : "ERR_ARGS_NUM", \ - "ERROR_OP" : "ERROR_OP", \ - "SUCCESS" : "SUCCESS", \ - } - -def openLog (): - global fLog - - try: - if fLog == None: - fLog = open (logFile, "a") - except IOError, e: - #print e - pass - -def log (str): - global fLog - - if fLog != None: - str = "[%s]:" % _asctime (_localtime()) + str + "\n" - fLog.write (str) - -def closeLog (): - global fLog - - if fLog != None: - fLog.close () - -def isProcessRun (pidFile, name): - try: - fpid = open (pidFile, "r") - pid = fpid.readline () - fpid.close () - except IOError, e: - return -1 - - pid = pid[:-1] - ps = os.popen ("ps -ae") - for l in ps: - if pid in l and name in l: - ps.close () - return 0 - - ps.close () - return -2 - -def isToolExist (name): - if _exists (name): - return 0 - return -1 - - -def checkvSwitch (): - global result - - ret = isProcessRun (vSwitchDBPidFile, vSwitchDBDaemonName); - if ret < 0: - if ret == -1: result = errors["NO_DB_PID_FILE"] - if ret == -2: result = errors["DB_NOT_RUN"] - return -1 - - ret = isProcessRun (vSwitchPidFile, vSwitchDaemonName) - if ret < 0: - if ret == -1: result = errors["NO_SWITCH_PID_FILE"] - if ret == -2: result = errors["SWITCH_NOT_RUN"] - return -1 - - if isToolExist (vsctlPath) < 0: - result = errors["NO_VSCTL"] - return -1 - - return 0 - -def doCmd (cmds, lines=False): - cmd = "" - for i in cmds: - cmd += " " - cmd += i - - log ("do command '%s'" % cmd) - f = os.popen (cmd) - if lines == True: - res = f.readlines () - else: - res = f.readline () - res = res[:-1] - f.close () - - if lines == False: - log ("command output '%s'" % res) - return res - -######################## GRE creation utils ########################## -# UUID's format is 8-4-4-4-12 -def isUUID (uuid): - list = uuid.split ("-") - - if len (list) != 5: - return -1 - - if len (list[0]) != 8 or len (list[1]) != 4 \ - or len (list[2]) != 4 or len (list[3]) != 4 \ - or len (list[4]) != 12: - return -1 - - return 0 - -def checkGREInterface (bridge, remoteIP, greKey): - ports = getPortsOnBridge(bridge) - if ports == None: - return 0 - - for i in ports: - ifaces = getInterfacesOnPort(i) - if ifaces == None: - continue - - for j in ifaces: - if j == '[]': - continue - options = getFieldOfInterface(j, "options") - if remoteIP in options and greKey in options: - log("WARNING: GRE tunnel for remote_ip=%s key=%s already here, \ -interface(%s)" % (remoteIP, greKey, j)) - return -1 - - return 0 - - -def createGRE (bridge, remoteIP, greKey): - global result - - name = "%sgre" % bridge - if checkGREInterface(bridge, remoteIP, greKey) < 0: - result = errors["TUNNEL_EXISTED"] - return 0 - - wait = [vsctlPath, "--timeout=30 wait-until bridge %s -- get bridge %s name" % \ - (bridge, bridge)] - res = doCmd(wait) - if bridge not in res: - log("WARNIING:Can't find bridge %s for creating tunnel!" % bridge) - result = errors["COMMAND_FAILED"] - return -1 - - createInterface = [vsctlPath, "create interface", "name=%s" % name, \ - 'type=gre options:"remote_ip=%s key=%s"' % (remoteIP, greKey)] - ifaceUUID = doCmd (createInterface) - if isUUID (ifaceUUID) < 0: - result = errors["COMMAND_FAILED"]; - return -1 - - createPort = [vsctlPath, "create port", "name=%s" % name, \ - "interfaces=[%s]" % ifaceUUID] - portUUID = doCmd (createPort) - if isUUID (portUUID) < 0: - result = errors["COMMAND_FAILED"]; - return -1 - - addBridge = [vsctlPath, "add bridge %s" % bridge, "ports %s" % portUUID] - doCmd (addBridge) - - wait = [vsctlPath, "--timeout=5 wait-until port %s -- get port %s name" % \ - (name, name)] - res = doCmd(wait) - if name in res: - result = errors["SUCCESS"] - return 0 - else: - result = errors["COMMAND_FAILED"] - return -1 -######################## End GRE creation utils ########################## - -######################## Flow creation utils ########################## -def getPortsOnBridge(bridge): - listBr = [vsctlPath, "list br", bridge] - res = doCmd(listBr, True) - - for i in res: - if "ports" in i: - (x, str) = i.split(":") - str = str.lstrip().rstrip() - str = str[1:] - str = str[:-1] - return str.split(",") - return None - -def getFieldOfPort(nameOruuid, field): - listport = [vsctlPath, "list port", nameOruuid] - res = doCmd(listport, True) - - for i in res: - if field in i: - (x, r) = i.split(":") - return r.lstrip().rstrip() - return None - -def getFieldOfInterface(nameOruuid, field): - listIface = [vsctlPath, "list interface", nameOruuid] - res = doCmd(listIface, True) - - for i in res: - if field in i: - (x, r) = i.split(":") - return r.lstrip().rstrip() - return None - -def strip(str, direction="default"): - str = str.lstrip().rstrip() - if direction == "left": - return str[1:] - if direction == "right": - return str[:-1] - if direction == "both": - str = str[1:] - str = str[:-1] - return str - return str - -def getVifPort(bridge, vifName): - portUuids = getPortsOnBridge(bridge) - if portUuids == None: - log("No ports on bridge %s" % bridge) - return None - - for i in portUuids: - name = getFieldOfPort(i, "name") - if name == None: - log("WARNING: no name found for %s" % name) - continue - - name = strip(name, "both") - if name == vifName: - return getFieldOfInterface(vifName, "ofport") - return None - -def getInterfacesOnPort(nameOruuid): - listPort = [vsctlPath, "list port", nameOruuid] - res = doCmd(listPort, True) - - for i in res: - if "interfaces" in i: - (x, str) = i.split(":") - str = strip(str, "both") - return str.split(",") - return None - -def getOfPortsByType(bridge, askGre): - portUuids = getPortsOnBridge(bridge) - if portUuids == None: - log("WARNING:No ports on bridge %s" % bridge) - return [] - - OfPorts = [] - for i in portUuids: - ifaces = getInterfacesOnPort(i) - if ifaces == None: - log("WARNING:No interfaces on port %s" % i) - continue - - for j in ifaces: - if j == '[]': - log("WARNING:invalid interface [] on port %s" % i) - continue - - type = getFieldOfInterface(j, "type") - type = strip(type) - if askGre == True and type == "gre": - ofport = getFieldOfInterface(j, "ofport") - if ofport != '[]':OfPorts.append(strip(ofport)) - elif askGre == False and type != "gre": - ofport = getFieldOfInterface(j, "ofport") - if ofport != '[]' and ofport != "65534":OfPorts.append(strip(ofport)) - return OfPorts - -def getNoneGreOfPort(bridge): - return getOfPortsByType(bridge, False) - -def getGreOfPorts(bridge): - return getOfPortsByType(bridge, True) - -def formatFlow(inPort, vlan, mac, outPut): - flow = "in_port=%s dl_vlan=%s dl_dst=%s idle_timeout=0 hard_timeout=0 \ - priority=10000 actions=strip_vlan,output:%s" % (inPort, vlan, mac, outPut) - return flow - -def formatDropFlow(inPort, vlan): - flow = "in_port=%s dl_vlan=%s priority=0 idle_timeout=0 hard_timeout=0 \ - actions=drop" % (inPort, vlan) - return flow - -def delFlow(mac): - param = "dl_dst=%s" % mac - delFlow = ["ovs-ofctl del-flows %s" % bridge, '"%s"' % param] - doCmd(delFlow) - -def delARPFlow(vlan): - param = "dl_type=0x0806 dl_vlan=%s" % vlan - delFlow = ["ovs-ofctl del-flows %s" % bridge, '"%s"' % param] - doCmd(delFlow) - -def delDHCPFlow(vlan): - param = "dl_type=0x0800 nw_proto=17 tp_dst=68 dl_vlan=%s" % vlan - delFlow = ["ovs-ofctl del-flows %s" % bridge, '"%s"' % param] - doCmd(delFlow) - -def delDHCPClientFlow(vlan): - param = "dl_type=0x0800 nw_proto=17 tp_dst=67 dl_vlan=%s" % vlan - delFlow = ["ovs-ofctl del-flows %s" % bridge, '"%s"' % param] - doCmd(delFlow) - -def delDropFlow(vlan): - param = "priority=0 dl_vlan=%s" % vlan - delFlow = ["ovs-ofctl del-flows %s" % bridge, '"%s"' % param] - doCmd(delFlow) - -def formatNormalFlow(): - flow = "priority=0 idle_timeout=0 hard_timeout=0 actions=normal" - return flow - -def formatDHCPFlow(bridge, inPort, vlan, ports): - outputs = '' - for i in ports: - str = "output:%s," % i - outputs += str - outputs = outputs[:-1] - flow = "in_port=%s dl_vlan=%s dl_type=0x0800 nw_proto=17 tp_dst=67 idle_timeout=0 hard_timeout=0 \ - priority=10000 actions=strip_vlan,%s" % (inPort, vlan, outputs) - return flow - -def formatDHCPClientFlow(bridge, inPort, vlan, ports): - outputs = '' - for i in ports: - str = "output:%s," % i - outputs += str - outputs = outputs[:-1] - flow = "in_port=%s dl_vlan=%s dl_type=0x0800 nw_proto=17 tp_dst=68 idle_timeout=0 hard_timeout=0 \ - priority=10000 actions=strip_vlan,%s" % (inPort, vlan, outputs) - return flow - -def formatARPFlow(bridge, inPort, vlan, ports): - outputs = '' - for i in ports: - str = "output:%s," % i - outputs += str - - outputs = outputs[:-1] - flow = "in_port=%s dl_vlan=%s dl_type=0x0806 idle_timeout=0 hard_timeout=0 \ - priority=10000 actions=strip_vlan,%s" % (inPort, vlan, outputs) - return flow - -def createFlow (bridge, vifName, mac, remap): - global result - inport = getGreOfPorts(bridge) - if len(inport) == 0: - log("WARNING: no inport found") - result = errors["NO_INPORT"] - return -1 - - output = getVifPort(bridge, vifName) - if output == None: - log("WARNING: cannot find ofport for %s" % vifName) - result = errors["NO_OFPORT"] - return -1 - if output == '[]': - log("WARNING: ofport is [] for %s" % vifName) - result = errors["NO_OFPORT"] - return -1 - - #set remap here, remap has format e.g. [1,22,200,13,16] - log("") - log("Create flow for remap") - noneGreOfPorts = getNoneGreOfPort(bridge) - isARP = True - if len(noneGreOfPorts) == 0: - log("WARNING: no none GRE ofports found, no ARP flow and DHCP flow will be created") - isARP = False - - for j in remap.split("/"): - for i in inport: - flow = formatDropFlow(i, j) - param = bridge + ' "%s"' % flow - dropflow = ["ovs-ofctl add-flow", param] - doCmd (dropflow) - - flow = formatFlow(i, j, mac, output) - param = bridge + ' "%s"' % flow - addflow = ["ovs-ofctl add-flow", param] - doCmd (addflow) - - - if isARP == True: - flow = formatARPFlow(bridge, i, j, noneGreOfPorts) - param = bridge + ' "%s"' % flow - addflow = ["ovs-ofctl add-flow", param] - doCmd (addflow) - - flow = formatDHCPFlow(bridge, i, j, noneGreOfPorts) - param = bridge + ' "%s"' % flow - addflow = ["ovs-ofctl add-flow", param] - doCmd (addflow) - - flow = formatDHCPClientFlow(bridge, i, j, noneGreOfPorts) - param = bridge + ' "%s"' % flow - addflow = ["ovs-ofctl add-flow", param] - doCmd (addflow) - - # add normal flow make switch work as L2/L3 switch - flow = formatNormalFlow() - param = bridge + ' "%s"' % flow - addflow = ["ovs-ofctl add-flow", param] - doCmd (addflow) - - result = errors["SUCCESS"] - return 0 -######################## End Flow creation utils ########################## - -def setTag(bridge, vifName, vlan): - # The startVM command is slow, we may wait for a while for it creates vif on - # open vswitch - log("Waiting for %s ..." % vifName) - waitPortCmd = [vsctlPath, "--timeout=10 wait-until port %s -- get port %s name" % \ - (vifName, vifName)] - doCmd (waitPortCmd) - log("%s is here" % vifName) - - if getVifPort(bridge, vifName) == None: - log("WARNING: %s is not on bridge %s" % (vifName, bridge)) - return 0 - - log("Set tag") - setTagCmd = [vsctlPath, "set port", vifName, "tag=%s"%vlan] - doCmd (setTagCmd) - return 0 - -def doCreateGRE(bridge, remoteIP, key): - global result - if createGRE(bridge, remoteIP, key) < 0: - log("create GRE tunnel on %s for %s failed" % (bridge, \ - remoteIP)) - else: - log("WARNING: create GRE tunnel on %s for %s success" % (bridge, \ - remoteIP)) - print result - -def doCreateFlow (bridge, vifName, mac, remap): - global result - if createFlow(bridge, vifName, mac, remap) < 0: - log ("Create flow failed(bridge=%s, vifName=%s, mac=%s,\ -remap=%s" % (bridge, vifName, mac, remap)) - else: - log ("Create flow success(bridge=%s, vifName=%s, mac=%s,\ -remap=%s" % (bridge, vifName, mac, remap)) - print result - -def doSetTag (bridge, vifName, tag): - setTag(bridge, vifName, tag) - -def doAskPorts(bridge, vifNames): - vifs = vifNames.split(",") - if len(vifs) == 0: - return ' ' - - ofports = [] - for vif in vifs: - op = getVifPort(bridge, vif) - if op == None: - log("doAskPorts: no port(bridge:%s, vif:%s)" % (bridge, vif)) - continue - ofports.append(op) - - return ",".join(ofports) - -def doDeleteFlow(bridge, ofports, macs, remap, reCreate): - for i in macs.split(","): - delFlow(i) - log("Delete flows for %s" % i) - - remap = strip(remap) - - if reCreate == "false": - for j in remap.split("/"): - delARPFlow(j) - delDHCPFlow(j) - delDHCPClientFlow(j) - delDropFlow(j) - return - - # remove our port from arp flow - inport = getGreOfPorts(bridge) - if len(inport) == 0: - log("WARNING:no inports") - return - - noneGreOfPorts = getNoneGreOfPort(bridge) - for i in ofports.split(","): - try: - noneGreOfPorts.remove(i) - except: - log("WARNING:ofport %s is not on bridge %s" % (i, bridge)) - log("Delete ARP flows for(ofport=%s)" % i) - - for j in remap.split("/"): - delARPFlow(j) - delDHCPFlow(j) - delDHCPClientFlow(j) - delDropFlow(j) - for i in inport: - flow = formatARPFlow(bridge, i, j, noneGreOfPorts) - param = bridge + ' "%s"' % flow - addflow = ["ovs-ofctl add-flow", param] - doCmd (addflow) - - flow = formatDHCPFlow(bridge, i, j, noneGreOfPorts) - param = bridge + ' "%s"' % flow - addflow = ["ovs-ofctl add-flow", param] - doCmd (addflow) - - flow = formatDHCPClientFlow(bridge, i, j, noneGreOfPorts) - param = bridge + ' "%s"' % flow - addflow = ["ovs-ofctl add-flow", param] - doCmd (addflow) - - # add normal flow make switch work as L2/L3 switch - flow = formatNormalFlow() - param = bridge + ' "%s"' % flow - addflow = ["ovs-ofctl add-flow", param] - doCmd (addflow) - -def checkArgNum(num): - if len (sys.argv) < num: - result = errors["ERR_ARGS_NUM"] - print result - log ("Error number of arguments") - sys.exit (-1) - -if __name__ == "__main__": - global result - - openLog () - - - if checkvSwitch () < 0: - print result - log ("Check switch failed, reason = '%s'" % result) - sys.exit (-1) - - op = sys.argv[1] - if op == "createGRE": - checkArgNum(5) - bridge = sys.argv[2] - remoteIP = sys.argv[3] - key = sys.argv[4] - doCreateGRE(bridge, remoteIP, key) - sys.exit(0) - elif op == "createFlow": - checkArgNum(6) - bridge = sys.argv[2] - vifName = sys.argv[3] - mac = sys.argv[4] - remap = sys.argv[5] - doCreateFlow(bridge, vifName, mac, remap) - sys.exit(0) - elif op == "deleteFlow": - checkArgNum(7) - bridge = sys.argv[2] - ofports = sys.argv[3] - macs = sys.argv[4] - remap = sys.argv[5] - reCreate = sys.argv[6] - doDeleteFlow(bridge, ofports, macs, remap, reCreate) - elif op == "setTag": - checkArgNum(5) - bridge = sys.argv[2] - vifName = sys.argv[3] - tag = sys.argv[4] - doSetTag(bridge, vifName, tag) - elif op == "askPorts": - checkArgNum(4) - bridge = sys.argv[2] - vifNames = sys.argv[3] - print doAskPorts(bridge, vifNames) - sys.exit(0) - else: - log("WARNING: get an unkown op %s" % op) - result=errors["ERROR_OP"] - print result - sys.exit(-1) - - result = errors["SUCCESS"] - closeLog () - print result diff --git a/scripts/vm/hypervisor/xenserver/vmops b/scripts/vm/hypervisor/xenserver/vmops index 6af2deced19..02e7702cf87 100755 --- a/scripts/vm/hypervisor/xenserver/vmops +++ b/scripts/vm/hypervisor/xenserver/vmops @@ -128,46 +128,6 @@ def ipassoc(session, args): return txt -@echo -def vlanRemapUtils(session, args): - cmd = [] - cmd.insert(0, "python") - cmd.insert(1, "/opt/xensource/bin/vlanRemapUtils.py") - - op = args.pop("op") - cmd.insert(2, op) - if op == "createGRE": - cmd.insert(3, args.pop("bridge")) - cmd.insert(4, args.pop("remoteIP")) - cmd.insert(5, args.pop("greKey")) - elif op == "createFlow": - cmd.insert(3, args.pop("bridge")) - cmd.insert(4, args.pop("vifName")) - cmd.insert(5, args.pop("mac")) - cmd.insert(7, args.pop("remap")) - elif op == "deleteFlow": - cmd.insert(3, args.pop("bridge")) - cmd.insert(4, args.pop("ofports")) - cmd.insert(5, args.pop("macs")) - cmd.insert(6, args.pop("remap")) - cmd.insert(7, args.pop("reCreate")) - elif op == "setTag": - cmd.insert(3, args.pop("bridge")) - cmd.insert(4, args.pop("vifName")) - cmd.insert(5, args.pop("tag")) - elif op == "askPorts": - cmd.insert(3, args.pop("bridge")) - cmd.insert(4, args.pop("vifNames")) - - - try: - txt = util.pread2(cmd) - except: - util.SMlog("vlanRemapUtils failed") - txt = 'failed' - - return txt - @echo def vm_data(session, args): router_ip = args.pop('routerIP') @@ -559,248 +519,6 @@ def default_ebtables_rules(vm_name, vif, vm_ip, vm_mac): util.SMlog("Failed to program default ebtables OUT rules") return 'false' -def format_ovs_vm_log_name(vmName): - vm_name = "ovs-%s" % vmName; - logfilename = "/var/run/cloud/" + vm_name +".log" - return logfilename - -def remove_ovs_log_for_vm(vmName): - logfilename = format_ovs_vm_log_name(vmName) - - result = True - try: - os.remove(logfilename) - except: - util.SMlog("Failed to delete ovs log file " + logfilename) - result = False - return result - -def ovs_get_info_from_log(vmName, num): - logfilename = format_ovs_vm_log_name(vmName) - try: - lines = [line.rstrip() for line in open(logfilename)] - return lines[num] - except: - util.SMlog("Failed to open ovs log %s" % logfilename); - remove_ovs_log_for_vm(vmName) - return None - -def ovs_get_common_info_from_log(vmName): - return ovs_get_info_from_log(vmName, 0) - -def ovs_get_nic_info_from_log(vmName): - return ovs_get_info_from_log(vmName, 1) - -def ovs_get_mac_info_from_log(vmName): - return ovs_get_info_from_log(vmName, 2) - -def ovs_get_vlans_info_from_log(vmName): - return ovs_get_info_from_log(vmName, 3) - -def ovs_get_ofports_info_from_log(vmName): - return ovs_get_info_from_log(vmName, 4) - -def ovs_parse_common_info_from_log(vmName, num): - info = ovs_get_common_info_from_log(vmName) - if info == None: - return None - return info.split(",")[num] - -def ovs_get_bridge_from_log(vmName): - return ovs_parse_common_info_from_log(vmName, 1) - -def ovs_get_vm_id_from_log(vmName): - return ovs_parse_common_info_from_log(vmName, 2) - -def ovs_get_seqno_from_log(vmName): - return ovs_parse_common_info_from_log(vmName, 3) - -def ovs_get_tag_from_log(vmName): - return ovs_parse_common_info_from_log(vmName, 4) - - -def ovs_handle_rebooted_vm(session, vmName): - curr_domid = '-1' - - (curr_domid, vifrs, hostuuid) = ovs_get_domid_vifrs_hostuuid(session, vmName) - - old_id = ovs_get_vm_id_from_log(vmName) - if curr_domid == old_id: - util.SMlog("OvsInfo:%s is normal" % vmName) - return True - - util.SMlog("%s rebooted, reset flow for it" % vmName) - try: - vlanstr = ovs_get_vlans_info_from_log(vmName) - bridge = ovs_get_bridge_from_log(vmName) - except Exception, e: - util.SMlog(e.__str__()) - util.SMlog("ovs get info from %s failed" % \ - format_ovs_vm_log_name(vmName)) - return False - - i = 0 - if vlanstr == None: - util.SMlog("OVSErr: cannot get vlanstr for %s" % vmName) - return False - - tag = ovs_get_tag_from_log(vmName) - nics = [] - macs = [] - for vifr in vifrs: - vifName = "vif" + curr_domid + "." + vifr[0] - vlanRemapUtils(session, {"op":"setTag", "vifName":vifName, "bridge":bridge, "tag":tag}) - vlanRemapUtils(session, {"op":"createFlow", "vifName":vifName, "bridge":bridge, "mac":vifr[1], "remap":vlanstr}) - nics.append(vifName) - macs.append(vifr[1]) - i += 1 - seqno = ovs_get_seqno_from_log(vmName) - ofports = ovs_get_ofports_info_from_log(vmName) - ovs_write_vm_log(bridge, vmName, curr_domid, seqno, nics, macs, tag, vlanstr, ofports) - - return True - -@echo -def ovs_get_vm_log(session, args): - host_uuid = args.pop('host_uuid') - try: - thishost = session.xenapi.host.get_by_uuid(host_uuid) - hostrec = session.xenapi.host.get_record(thishost) - vms = hostrec.get('resident_VMs') - except Exception, e: - util.SMlog("Failed to get host from uuid %s, exception: %s" % (host_uuid, e.__str__())) - return ' ' - - result = [] - try: - for name in [session.xenapi.VM.get_name_label(x) for x in vms]: - if 1 not in [ name.startswith(c) for c in ['r-', 'i-'] ]: - continue - ovs_handle_rebooted_vm(session, name) - if name.startswith('i-'): - info = ovs_get_common_info_from_log(name) - if info == None: - util.SMlog("ovs_get_common_info_from_log return None for %s" % name) - continue - result.append(info) - except Exception, e: - util.SMlog(e.__str__()) - util.SMlog("OVs failed to get logs, better luck next time!") - - return ";".join(result) - -def ovs_write_vm_log(bridge, vmName, vmId, seqno, vifNames, macs, tag, vlans, ofports): - logfilename = format_ovs_vm_log_name(vmName) - - util.SMlog("Writing ovs log to " + logfilename) - logf = open(logfilename, 'w') - output = ','.join([vmName, bridge, vmId, seqno, tag]) - result = True - try: - logf.write(output) - logf.write('\n') - output = ','.join(vifNames) - logf.write(output) - logf.write('\n') - output = ','.join(macs) - logf.write(output) - logf.write('\n') - logf.write(vlans) - logf.write('\n') - logf.write(ofports) - logf.write('\n') - except: - util.SMlog("Failed to write to ovs log file " + logfilename) - result = False - - logf.close() - - return result - -def ovs_delete_flow(session, args): - bridge = args.pop('bridge') - vm_name = args.pop('vmName') - reCreate = args.pop('reCreate') - - macs = ovs_get_mac_info_from_log(vm_name) - vlans = ovs_get_vlans_info_from_log(vm_name) - ofports = ovs_get_ofports_info_from_log(vm_name) - - if macs == None or vlans == None or ofports == None: - return 'ERROR_LOG' - - vlanRemapUtils(session, {"op":"deleteFlow", "bridge":bridge, \ - "ofports":ofports, "macs":macs, "remap":vlans, "reCreate":reCreate}) - - remove_ovs_log_for_vm(vm_name) - return 'SUCCESS' - -def ovs_get_domid_vifrs_hostuuid(session, vm_name): - def get_vif_field(name, field): - return session.xenapi.VIF.get_record(name).get(field) - - try: - vm = session.xenapi.VM.get_by_name_label(vm_name) - if len(vm) != 1: - return 'NO_VM' - vm_rec = session.xenapi.VM.get_record(vm[0]) - vm_vifs = vm_rec.get('VIFs') - vifrs = [] - for vif in vm_vifs: - rec = (get_vif_field(vif, 'device'), get_vif_field(vif, 'MAC')) - vifrs.append(rec) - domid = vm_rec.get('domid') - host = vm_rec.get('resident_on') - host_rec = session.xenapi.host.get_record(host) - uuid = host_rec.get('uuid') - util.SMlog("OVSINFO: (domid:%s, uuid:%s)" % (domid, uuid)) - return (domid, vifrs, uuid) - - except: - util.SMlog("### Failed to get domid or vif list for vm ##" + vm_name) - return (-1, [], "-1") - -def ovs_set_tag_and_flow(session, args): - bridge = args.pop('bridge') - vm_name = args.pop('vmName') - vlanStr = args.pop('vlans') - tag = args.pop('tag') - seqno = args.pop('seqno') - - (domid, vifrs, hostuuid) = ovs_get_domid_vifrs_hostuuid(session, vm_name) - - if domid == '-1': - util.SMlog("### Failed to get domid for vm (-1): " + vm_name) - return 'NO_DOMID' - - if len(vifrs) == 0: - return 'SUCCESS' - - #delete old flows first - ovs_delete_flow(session, {"bridge":bridge, "vmName":vm_name, "reCreate":"false"}) - - if vlanStr.startswith("/"): vlanStr = vlanStr[1:] - if vlanStr.endswith("/"): vlanStr = vlanStr[:-1] - vlans = vlanStr.split("/") - vifNames = [] - macs = [] - for vifr in vifrs: - vifName = "vif" + domid + "." + vifr[0] - vifNames.append(vifName) - mac = vifr[1] - macs.append(mac) - vlanRemapUtils(session, {"op":"setTag", "vifName":vifName, "bridge":bridge, "tag":tag}) - vlanRemapUtils(session, {"op":"createFlow", "vifName":vifName, "bridge":bridge, "mac":mac, "remap":vlanStr}) - - ofports = vlanRemapUtils(session, {"op":"askPorts", "bridge":bridge, "vifNames":",".join(vifNames)}) - res = ovs_write_vm_log(bridge, vm_name, domid, seqno, vifNames, macs, tag, vlanStr, ofports) - if res == 'false': - return 'CREATE_LOG_FAILED' - - #see if there is rebooted vm to handle - ovs_get_vm_log(session, {"host_uuid":hostuuid}) - return 'SUCCESS' - @echo def default_network_rules_systemvm(session, args): vm_name = args.pop('vmName') @@ -1314,5 +1032,5 @@ def network_rules(session, args): if __name__ == "__main__": - XenAPIPlugin.dispatch({"pingtest": pingtest, "setup_iscsi":setup_iscsi, "gethostvmstats": gethostvmstats, "getvncport": getvncport, "getgateway": getgateway, "preparemigration": preparemigration, "setIptables": setIptables, "pingdomr": pingdomr, "pingxenserver": pingxenserver, "ipassoc": ipassoc, "vm_data": vm_data, "savePassword": savePassword, "saveDhcpEntry": saveDhcpEntry, "setFirewallRule": setFirewallRule, "setLoadBalancerRule": setLoadBalancerRule, "createFile": createFile, "deleteFile": deleteFile, "networkUsage": networkUsage, "network_rules":network_rules, "can_bridge_firewall":can_bridge_firewall, "default_network_rules":default_network_rules, "destroy_network_rules_for_vm":destroy_network_rules_for_vm, "default_network_rules_systemvm":default_network_rules_systemvm, "get_rule_logs_for_vms":get_rule_logs_for_vms, "setLinkLocalIP":setLinkLocalIP, "lt2p_vpn":lt2p_vpn, "vlanRemapUtils":vlanRemapUtils, "ovs_set_tag_and_flow":ovs_set_tag_and_flow, "ovs_get_vm_log":ovs_get_vm_log,"ovs_delete_flow":ovs_delete_flow,"cleanup_rules":cleanup_rules}) + XenAPIPlugin.dispatch({"pingtest": pingtest, "setup_iscsi":setup_iscsi, "gethostvmstats": gethostvmstats, "getvncport": getvncport, "getgateway": getgateway, "preparemigration": preparemigration, "setIptables": setIptables, "pingdomr": pingdomr, "pingxenserver": pingxenserver, "ipassoc": ipassoc, "vm_data": vm_data, "savePassword": savePassword, "saveDhcpEntry": saveDhcpEntry, "setFirewallRule": setFirewallRule, "setLoadBalancerRule": setLoadBalancerRule, "createFile": createFile, "deleteFile": deleteFile, "networkUsage": networkUsage, "network_rules":network_rules, "can_bridge_firewall":can_bridge_firewall, "default_network_rules":default_network_rules, "destroy_network_rules_for_vm":destroy_network_rules_for_vm, "default_network_rules_systemvm":default_network_rules_systemvm, "get_rule_logs_for_vms":get_rule_logs_for_vms, "setLinkLocalIP":setLinkLocalIP, "lt2p_vpn":lt2p_vpn, "cleanup_rules":cleanup_rules}) diff --git a/scripts/vm/hypervisor/xenserver/xenserver56fp1/patch b/scripts/vm/hypervisor/xenserver/xenserver56fp1/patch index 67c20b97d3c..8042e30d103 100644 --- a/scripts/vm/hypervisor/xenserver/xenserver56fp1/patch +++ b/scripts/vm/hypervisor/xenserver/xenserver56fp1/patch @@ -12,6 +12,7 @@ NFSSR.py=/opt/xensource/sm nfs.py=/opt/xensource/sm vmops=..,0755,/etc/xapi.d/plugins +ovsgre=..,0755,/etc/xapi.d/plugins vmopsSnapshot=..,0755,/etc/xapi.d/plugins hostvmstats.py=..,0755,/opt/xensource/sm systemvm.iso=../../../../../vms,0644,/opt/xensource/packages/iso @@ -31,4 +32,3 @@ call_firewall.sh=../../../../network/domr/,0755,/opt/xensource/bin call_loadbalancer.sh=../../../../network/domr/,0755,/opt/xensource/bin l2tp_vpn.sh=../../../../network/domr/,0755,/opt/xensource/bin cloud-setup-bonding.sh=..,0755,/opt/xensource/bin -vlanRemapUtils.py=..,0755,/opt/xensource/bin diff --git a/server/src/com/cloud/configuration/DefaultComponentLibrary.java b/server/src/com/cloud/configuration/DefaultComponentLibrary.java index 94536cdeb3d..5b64912c11b 100644 --- a/server/src/com/cloud/configuration/DefaultComponentLibrary.java +++ b/server/src/com/cloud/configuration/DefaultComponentLibrary.java @@ -73,6 +73,7 @@ import com.cloud.network.dao.RemoteAccessVpnDaoImpl; import com.cloud.network.dao.VpnUserDaoImpl; import com.cloud.network.lb.LoadBalancingRulesManagerImpl; import com.cloud.network.ovs.OvsNetworkManagerImpl; +import com.cloud.network.ovs.dao.GreTunnelDaoImpl; import com.cloud.network.ovs.dao.OvsWorkDaoImpl; import com.cloud.network.ovs.dao.VlanMappingDaoImpl; import com.cloud.network.ovs.dao.VlanMappingDirtyDaoImpl; @@ -245,6 +246,7 @@ public class DefaultComponentLibrary implements ComponentLibrary { addDao("VlanMappingDirtyDao", VlanMappingDirtyDaoImpl.class); addDao("OvsWorkDao", OvsWorkDaoImpl.class); addDao("VmFlowLogDao", VmFlowLogDaoImpl.class); + addDao("GreTunnelDao", GreTunnelDaoImpl.class); } Map> _managers = new HashMap>(); diff --git a/server/src/com/cloud/network/ovs/GreTunnelException.java b/server/src/com/cloud/network/ovs/GreTunnelException.java new file mode 100644 index 00000000000..b88baee0259 --- /dev/null +++ b/server/src/com/cloud/network/ovs/GreTunnelException.java @@ -0,0 +1,7 @@ +package com.cloud.network.ovs; + +public class GreTunnelException extends Exception { + public GreTunnelException(String msg) { + super(msg); + } +} diff --git a/server/src/com/cloud/network/ovs/OvsListener.java b/server/src/com/cloud/network/ovs/OvsListener.java index 4915d4ddfc0..df31e6253b0 100644 --- a/server/src/com/cloud/network/ovs/OvsListener.java +++ b/server/src/com/cloud/network/ovs/OvsListener.java @@ -15,6 +15,8 @@ import com.cloud.agent.api.StartupCommand; import com.cloud.exception.ConnectionException; import com.cloud.host.HostVO; import com.cloud.host.Status; +import com.cloud.network.ovs.dao.GreTunnelDao; +import com.cloud.network.ovs.dao.GreTunnelVO; import com.cloud.network.ovs.dao.OvsWorkDao; import com.cloud.network.ovs.dao.OvsWorkVO.Step; @@ -22,42 +24,62 @@ public class OvsListener implements Listener { public static final Logger s_logger = Logger.getLogger(OvsListener.class.getName()); OvsNetworkManager _ovsNetworkMgr; OvsWorkDao _workDao; + GreTunnelDao _tunnelDao; - public OvsListener(OvsNetworkManager ovsMgr, OvsWorkDao workDao) { + public OvsListener(OvsNetworkManager ovsMgr, OvsWorkDao workDao, GreTunnelDao tunnelDao) { this._ovsNetworkMgr = ovsMgr; this._workDao = workDao; + this._tunnelDao = tunnelDao; } @Override public boolean processAnswers(long agentId, long seq, Answer[] answers) { Set failedFlowVms = new HashSet(); - - for (Answer ans: answers) { - if (ans instanceof OvsCreateGreTunnelAnswer) { - OvsCreateGreTunnelAnswer r = (OvsCreateGreTunnelAnswer)ans; - String s = String.format("(hostIP:%1$s, remoteIP:%2$s, bridge:%3$s, greKey:%4$s)", - r.getHostIp(), r.getRemoteIp(), - r.getBridge(), r.getKey()); - if (!r.getResult()) { - s_logger.warn("Create GRE tunnel failed due to " + r.getDetails() + s); - } else { - s_logger.info("Create GRE tunnel success" + s); - } - } else if (ans instanceof OvsSetTagAndFlowAnswer) { - OvsSetTagAndFlowAnswer r = (OvsSetTagAndFlowAnswer)ans; - if (!r.getResult()) { - s_logger.warn("Failed to set flow for VM " + r.getVmId()); - _workDao.updateStep(r.getVmId(), r.getSeqNo(), Step.Error); - failedFlowVms.add(r.getVmId()); - } else { - s_logger.info("Success to set flow for VM " + r.getVmId()); - _workDao.updateStep(r.getVmId(), r.getSeqNo(), Step.Done); + try { + for (Answer ans : answers) { + if (ans instanceof OvsCreateGreTunnelAnswer) { + OvsCreateGreTunnelAnswer r = (OvsCreateGreTunnelAnswer) ans; + String s = String + .format("(hostIP:%1$s, remoteIP:%2$s, bridge:%3$s, greKey:%4$s)", + r.getHostIp(), r.getRemoteIp(), + r.getBridge(), r.getKey()); + if (!r.getResult()) { + s_logger.warn("Create GRE tunnel failed due to " + + r.getDetails() + s); + } else { + GreTunnelVO tunnel = _tunnelDao.getByFromAndTo( + r.getFrom(), r.getTo()); + if (tunnel == null) { + throw new GreTunnelException( + "No record matches from = " + r.getFrom() + " to = " + r.getTo()); + } else { + tunnel.setInPort(r.getPort()); + _tunnelDao.update(tunnel.getId(), tunnel); + s_logger.info("Create GRE tunnel success" + s + + " from " + r.getFrom() + " to " + + r.getTo() + " inport=" + r.getPort()); + } + } + } else if (ans instanceof OvsSetTagAndFlowAnswer) { + OvsSetTagAndFlowAnswer r = (OvsSetTagAndFlowAnswer) ans; + if (!r.getResult()) { + s_logger.warn("Failed to set flow for VM " + + r.getVmId()); + _workDao.updateStep(r.getVmId(), r.getSeqNo(), + Step.Error); + failedFlowVms.add(r.getVmId()); + } else { + s_logger.info("Success to set flow for VM " + + r.getVmId()); + _workDao.updateStep(r.getVmId(), r.getSeqNo(), + Step.Done); + } } + // TODO: handle delete failure } - - //TODO: handle delete failure + } catch (Exception e) { + e.printStackTrace(); } - if (failedFlowVms.size() > 0) { _ovsNetworkMgr.scheduleFlowUpdateToHosts(failedFlowVms, false, new Long(10*1000l)); } diff --git a/server/src/com/cloud/network/ovs/OvsNetworkManager.java b/server/src/com/cloud/network/ovs/OvsNetworkManager.java index dae467e77bb..6931886ca04 100644 --- a/server/src/com/cloud/network/ovs/OvsNetworkManager.java +++ b/server/src/com/cloud/network/ovs/OvsNetworkManager.java @@ -16,12 +16,8 @@ import com.cloud.vm.VirtualMachineProfile; public interface OvsNetworkManager extends Manager { public boolean isOvsNetworkEnabled(); - public long askVlanId(long accountId, long hostId); - - public String getVlanMapping(long accountId); - public void UserVmCheckAndCreateTunnel(Commands cmds, - VirtualMachineProfile profile, DeployDestination dest); + VirtualMachineProfile profile, DeployDestination dest) throws GreTunnelException; public void applyDefaultFlowToUserVm(Commands cmds, VirtualMachineProfile profile, DeployDestination dest); @@ -34,7 +30,7 @@ public interface OvsNetworkManager extends Manager { public void RouterCheckAndCreateTunnel(Commands cmds, VirtualMachineProfile profile, - DeployDestination dest); + DeployDestination dest) throws GreTunnelException; public void fullSync(List> states); diff --git a/server/src/com/cloud/network/ovs/OvsNetworkManagerImpl.java b/server/src/com/cloud/network/ovs/OvsNetworkManagerImpl.java index afbf7795f07..09638b16a4d 100644 --- a/server/src/com/cloud/network/ovs/OvsNetworkManagerImpl.java +++ b/server/src/com/cloud/network/ovs/OvsNetworkManagerImpl.java @@ -24,6 +24,8 @@ import com.cloud.exception.AgentUnavailableException; import com.cloud.host.HostVO; import com.cloud.host.dao.HostDao; import com.cloud.network.dao.NetworkDao; +import com.cloud.network.ovs.dao.GreTunnelDao; +import com.cloud.network.ovs.dao.GreTunnelVO; import com.cloud.network.ovs.dao.OvsWorkDao; import com.cloud.network.ovs.dao.OvsWorkVO; import com.cloud.network.ovs.dao.OvsWorkVO.Step; @@ -69,6 +71,7 @@ public class OvsNetworkManagerImpl implements OvsNetworkManager { @Inject UserVmDao _userVMDao; @Inject VMInstanceDao _instanceDao; @Inject AccountDao _accountDao; + @Inject GreTunnelDao _tunnelDao; String _name; boolean _isEnabled; ScheduledExecutorService _executorPool; @@ -109,9 +112,16 @@ public class OvsNetworkManagerImpl implements OvsNetworkManager { _serverId = ((ManagementServer)ComponentLocator.getComponent(ManagementServer.Name)).getId(); _executorPool = Executors.newScheduledThreadPool(10, new NamedThreadFactory("OVS")); _cleanupExecutor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("OVS-Cleanup")); - _ovsListener = new OvsListener(this, _workDao); + _ovsListener = new OvsListener(this, _workDao, _tunnelDao); _agentMgr.registerForHostEvents(_ovsListener, true, true, true); + //FIXME: + GreTunnelVO t = _tunnelDao.lockRow(new Long(1), true); + if (t == null) { + t = new GreTunnelVO(0, 0); + _tunnelDao.persist(t); + } + return true; } @@ -192,9 +202,6 @@ public class OvsNetworkManagerImpl implements OvsNetworkManager { } } - String vlans = getVlanMapping(vm.getAccountId()); - String tag = Long.toString(_vlanMappingDao.findByAccountIdAndHostId( - vm.getAccountId(), vm.getHostId()).getVlan()); Long agentId = null; VmFlowLogVO log = _flowLogDao.findByVmId(userVmId); if (log == null) { @@ -206,12 +213,16 @@ public class OvsNetworkManagerImpl implements OvsNetworkManager { if (vm != null && vm.getState() == State.Running) { agentId = vm.getHostId(); if (agentId != null ) { - OvsSetTagAndFlowCommand cmd = new OvsSetTagAndFlowCommand( - vm.getName(), tag, vlans, seqnum.toString(), vm.getId()); - Commands cmds = new Commands(cmd); + String vlans = getVlanInPortMapping(vm.getAccountId(), vm.getHostId()); + String tag = Long.toString(_vlanMappingDao.findByAccountIdAndHostId( + vm.getAccountId(), vm.getHostId()).getVlan()); + + Commands cmds = new Commands(new OvsSetTagAndFlowCommand( + vm.getName(), tag, vlans, seqnum.toString(), + vm.getId())); + try { _agentMgr.send(agentId, cmds, _ovsListener); - // TODO: clean dirty in answerListener } catch (AgentUnavailableException e) { s_logger.debug("Unable to send updates for vm: " + userVmId + "(agentid=" + agentId + ")"); @@ -239,9 +250,8 @@ public class OvsNetworkManagerImpl implements OvsNetworkManager { } //TODO: think about lock - @Override @DB - public long askVlanId(long accountId, long hostId) { + protected long askVlanId(long accountId, long hostId) { assert _isEnabled : "Who call me ??? while OvsNetwokr is not enabled!!!"; final Transaction txn = Transaction.currentTxn(); txn.start(); @@ -287,33 +297,9 @@ public class OvsNetworkManagerImpl implements OvsNetworkManager { return vlan; } - @Override @DB - public String getVlanMapping(long accountId) { - assert _isEnabled : "Who call me ??? while OvsNetwork is not enabled!!!"; - final Transaction txn = Transaction.currentTxn(); - txn.start(); - - List ours = _vlanMappingDao.listByAccountId(accountId); - txn.commit(); - - ArrayListvlans = new ArrayList(); - for (VlanMappingVO vo : ours) { - vlans.add(new Long(vo.getVlan())); - } - - assert vlans.size() > 0 : "Vlan map can't be null"; - StringBuffer buf = new StringBuffer(); - buf.append("/"); - for (Long i : vlans) { - buf.append(i.toString()); - buf.append("/"); - } - return buf.toString(); - } - - protected void CheckAndCreateTunnel(Commands cmds, VMInstanceVO instance, - DeployDestination dest) { + protected void CheckAndCreateTunnel(VMInstanceVO instance, + DeployDestination dest) throws GreTunnelException { if (!_isEnabled) { return; } @@ -325,33 +311,62 @@ public class OvsNetworkManagerImpl implements OvsNetworkManager { long hostId = dest.getHost().getId(); long accountId = instance.getAccountId(); - List vms = _userVmDao.listByAccountIdAndHostId(accountId, hostId); - if (vms.size() > 1 || (vms.size() == 1 && vms.get(0).getId() != instance.getId())) { - s_logger.debug("Already has GRE tunnel for account " + accountId - + " for host " + hostId); - return; + + final Transaction txn = Transaction.currentTxn(); + txn.start(); + //TODO: considerate router? + Listvms = _userVmDao.listByAccountId(accountId); + ListtoHostIds = new ArrayList(); + ListfromHostIds = new ArrayList(); + GreTunnelVO tvo = _tunnelDao.acquireInLockTable(new Long(1)); + if (tvo == null) { + throw new GreTunnelException("can't lock gre tunnel table for: from=" + hostId); } - vms = _userVmDao.listByAccountId(accountId); - ListremoteHostIds = new ArrayList(); for (UserVmVO v : vms) { Long rh = v.getHostId(); if (rh == null || rh.longValue() == hostId) { continue; } - if (!remoteHostIds.contains(rh)) { - remoteHostIds.add(rh); + GreTunnelVO tunnel = _tunnelDao.getByFromAndTo(hostId, rh.longValue()); + if (tunnel == null) { + tunnel = new GreTunnelVO(hostId, rh.longValue()); + _tunnelDao.persist(tunnel); + + if (!toHostIds.contains(rh)) { + toHostIds.add(rh); + } } + + tunnel = _tunnelDao.getByFromAndTo(rh.longValue(), hostId); + if (tunnel == null) { + tunnel = new GreTunnelVO(rh.longValue(), hostId); + _tunnelDao.persist(tunnel); + + if (!fromHostIds.contains(rh)) { + fromHostIds.add(rh); + } + } + } + _tunnelDao.releaseFromLockTable(new Long(1)); + txn.commit(); try { String myIp = dest.getHost().getPrivateIpAddress(); - for (Long i : remoteHostIds) { + for (Long i : toHostIds) { HostVO rHost = _hostDao.findById(i.longValue()); - cmds.addCommand( - 0, new OvsCreateGreTunnelCommand(rHost.getPrivateIpAddress(), "1")); - Commands cmd2s = new Commands( new OvsCreateGreTunnelCommand(myIp, "1")); + Commands cmds = new Commands( + new OvsCreateGreTunnelCommand( + rHost.getPrivateIpAddress(), "1", hostId, + i.longValue())); + _agentMgr.send(hostId, cmds , _ovsListener); + s_logger.debug("Ask host " + hostId + " to create gre tunnel to " + i.longValue()); + } + + for (Long i : fromHostIds) { + Commands cmd2s = new Commands(new OvsCreateGreTunnelCommand(myIp, "1", i.longValue(), hostId)); _agentMgr.send(i.longValue(), cmd2s , _ovsListener); s_logger.debug("Ask host " + i.longValue() + " to create gre tunnel to " + hostId); } @@ -360,6 +375,29 @@ public class OvsNetworkManagerImpl implements OvsNetworkManager { } } + @DB + protected String getVlanInPortMapping(long accountId, long from) { + List tunnels = _tunnelDao.getByFrom(from); + if (tunnels.size() == 0) { + return "[]"; + } else { + Transaction txn = Transaction.currentTxn(); + txn.start(); + List maps = new ArrayList(); + for (GreTunnelVO t : tunnels) { + VlanMappingVO m = _vlanMappingDao.lockByAccountIdAndHostId(accountId, t.getTo()); + if (m == null) { + s_logger.debug("Host " + t.getTo() + " has no VM for account " + accountId + ", skip it"); + continue; + } + String s = String.format("%1$s:%2$s", m.getVlan(), t.getInPort()); + maps.add(s); + } + txn.commit(); + return maps.toString(); + } + } + protected void applyDefaultFlow(Commands cmds, VMInstanceVO instance, DeployDestination dest) { if (!_isEnabled) { @@ -372,12 +410,16 @@ public class OvsNetworkManagerImpl implements OvsNetworkManager { return; } - String tag = Long.toString(askVlanId(instance.getAccountId(), instance.getHostId())); - String vlans = getVlanMapping(instance.getAccountId()); + long hostId = instance.getHostId(); + long accountId = instance.getAccountId(); + String tag = Long.toString(askVlanId(accountId, hostId)); CheckAndUpdateDhcpFlow(instance); - VmFlowLogVO log = _flowLogDao.findOrNewByVmId(instance.getId(), instance.getName()); - cmds.addCommand(new OvsSetTagAndFlowCommand(instance.getName(), tag, vlans, - Long.toString(log.getLogsequence()), instance.getId())); + String vlans = getVlanInPortMapping(accountId, hostId); + + VmFlowLogVO log = _flowLogDao.findOrNewByVmId(instance.getId(), + instance.getName()); + cmds.addCommand(new OvsSetTagAndFlowCommand(instance.getName(), tag, + vlans, Long.toString(log.getLogsequence()), instance.getId())); } //FIXME: if router has record in database but not start, this will hang 10 secs due to host @@ -402,14 +444,16 @@ public class OvsNetworkManagerImpl implements OvsNetworkManager { } try { - String vlans = getVlanMapping(accountId); - String tag = Long.toString(_vlanMappingDao.findByAccountIdAndHostId(router.getAccountId(), + String tag = Long.toString(_vlanMappingDao.findByAccountIdAndHostId(accountId, router.getHostId()).getVlan()); + long hostId = router.getHostId(); VmFlowLogVO log = _flowLogDao.findOrNewByVmId(instance.getId(), instance.getName()); + String vlans = getVlanInPortMapping(accountId, hostId); s_logger.debug("ask router " + router.getName() + " on host " - + router.getHostId() + " update vlan map to " + vlans); - _agentMgr.send(router.getHostId(), new OvsSetTagAndFlowCommand( - router.getName(), tag, vlans, Long.toString(log.getLogsequence()), instance.getId())); + + hostId + " update vlan map to " + vlans); + Commands cmds = new Commands(new OvsSetTagAndFlowCommand( + router.getName(), tag, vlans, Long.toString(log.getLogsequence()), instance.getId())); + _agentMgr.send(router.getHostId(), cmds, _ovsListener); } catch (Exception e) { e.printStackTrace(); } @@ -568,15 +612,15 @@ public class OvsNetworkManagerImpl implements OvsNetworkManager { @Override public void UserVmCheckAndCreateTunnel(Commands cmds, - VirtualMachineProfile profile, DeployDestination dest) { - CheckAndCreateTunnel(cmds, (VMInstanceVO)profile.getVirtualMachine(), dest); + VirtualMachineProfile profile, DeployDestination dest) throws GreTunnelException { + CheckAndCreateTunnel((VMInstanceVO)profile.getVirtualMachine(), dest); } @Override public void RouterCheckAndCreateTunnel(Commands cmds, VirtualMachineProfile profile, - DeployDestination dest) { - CheckAndCreateTunnel(cmds, (VMInstanceVO)profile.getVirtualMachine(), dest); + DeployDestination dest) throws GreTunnelException { + CheckAndCreateTunnel((VMInstanceVO)profile.getVirtualMachine(), dest); } @Override diff --git a/server/src/com/cloud/network/ovs/dao/GreTunnelDao.java b/server/src/com/cloud/network/ovs/dao/GreTunnelDao.java new file mode 100644 index 00000000000..753c70fcac2 --- /dev/null +++ b/server/src/com/cloud/network/ovs/dao/GreTunnelDao.java @@ -0,0 +1,10 @@ +package com.cloud.network.ovs.dao; + +import java.util.List; + +import com.cloud.utils.db.GenericDao; + +public interface GreTunnelDao extends GenericDao { + List getByFrom(long from); + GreTunnelVO getByFromAndTo(long from, long To); +} diff --git a/server/src/com/cloud/network/ovs/dao/GreTunnelDaoImpl.java b/server/src/com/cloud/network/ovs/dao/GreTunnelDaoImpl.java new file mode 100644 index 00000000000..6caa68674c5 --- /dev/null +++ b/server/src/com/cloud/network/ovs/dao/GreTunnelDaoImpl.java @@ -0,0 +1,44 @@ +package com.cloud.network.ovs.dao; + +import java.util.List; + +import javax.ejb.Local; + +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; +import com.cloud.utils.db.SearchCriteria.Op; + +@Local(value = { GreTunnelDao.class }) +public class GreTunnelDaoImpl extends GenericDaoBase + implements GreTunnelDao { + protected final SearchBuilder fromSearch; + protected final SearchBuilder fromToSearch; + + public GreTunnelDaoImpl() { + fromSearch = createSearchBuilder(); + fromSearch.and("from", fromSearch.entity().getFrom(), Op.EQ); + fromSearch.done(); + + fromToSearch = createSearchBuilder(); + fromToSearch.and("from", fromToSearch.entity().getFrom(), Op.EQ); + fromToSearch.and("to", fromToSearch.entity().getTo(), Op.EQ); + fromToSearch.done(); + } + + @Override + public List getByFrom(long from) { + SearchCriteria sc = fromSearch.create(); + sc.setParameters("from", from); + return listBy(sc, null); + } + + @Override + public GreTunnelVO getByFromAndTo(long from, long to) { + SearchCriteria sc = fromToSearch.create(); + sc.setParameters("from", from); + sc.setParameters("to", to); + return findOneBy(sc); + } + +} diff --git a/server/src/com/cloud/network/ovs/dao/GreTunnelVO.java b/server/src/com/cloud/network/ovs/dao/GreTunnelVO.java new file mode 100644 index 00000000000..94697e55a60 --- /dev/null +++ b/server/src/com/cloud/network/ovs/dao/GreTunnelVO.java @@ -0,0 +1,64 @@ +package com.cloud.network.ovs.dao; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; + +@Entity +@Table(name=("ovs_tunnel_alloc")) +public class GreTunnelVO { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "id") + private long id; + + @Column(name = "from") + private long from; + + @Column(name = "to") + private long to; + + @Column(name = "in_port") + private int inPort; + + public GreTunnelVO() { + + } + + public GreTunnelVO(long from, long to) { + this.from = from; + this.to = to; + this.inPort = 0; + } + + public GreTunnelVO(long id, long from, long to) { + this.from = from; + this.to = to; + this.inPort = 0; + this.id = id; + } + + public void setInPort(int port) { + inPort = port; + } + + public long getFrom() { + return from; + } + + public long getTo() { + return to; + } + + public int getInPort() { + return inPort; + } + + public long getId() { + return id; + } + +} diff --git a/server/src/com/cloud/network/ovs/dao/VlanMappingDao.java b/server/src/com/cloud/network/ovs/dao/VlanMappingDao.java index 43a9926ec6e..d74bd3e41a4 100644 --- a/server/src/com/cloud/network/ovs/dao/VlanMappingDao.java +++ b/server/src/com/cloud/network/ovs/dao/VlanMappingDao.java @@ -8,5 +8,7 @@ public interface VlanMappingDao extends GenericDao { List listByAccountIdAndHostId(long accountId, long hostId); List listByHostId(long hostId); List listByAccountId(long accountId); + List lockByAccountId(long accoutnId); VlanMappingVO findByAccountIdAndHostId(long accountId, long hostId); + VlanMappingVO lockByAccountIdAndHostId(long accountId, long hostId); } diff --git a/server/src/com/cloud/network/ovs/dao/VlanMappingDaoImpl.java b/server/src/com/cloud/network/ovs/dao/VlanMappingDaoImpl.java index 14cb19c1628..59d969cf076 100644 --- a/server/src/com/cloud/network/ovs/dao/VlanMappingDaoImpl.java +++ b/server/src/com/cloud/network/ovs/dao/VlanMappingDaoImpl.java @@ -59,9 +59,30 @@ public class VlanMappingDaoImpl extends GenericDaoBase @Override public VlanMappingVO findByAccountIdAndHostId(long accountId, long hostId) { + return getByAccountIdAndHostId(accountId, hostId, false); + } + + @Override + public List lockByAccountId(long accountId) { + SearchCriteria sc = accountIdSearch.create(); + sc.setParameters("account_id", accountId); + return lockRows(sc, null, true); + } + + @Override + public VlanMappingVO lockByAccountIdAndHostId(long accountId, long hostId) { + return getByAccountIdAndHostId(accountId, hostId, true); + } + + private VlanMappingVO getByAccountIdAndHostId(long accountId, long hostId, boolean lock) { SearchCriteria sc = accountHostSearch.create(); sc.setParameters("account_id", accountId); sc.setParameters("host_id", hostId); - return findOneBy(sc); + + if (!lock) { + return findOneBy(sc); + } else { + return lockOneRandomRow(sc, true); + } } } diff --git a/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java b/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java index bf3d40d400f..9566ed0b46b 100644 --- a/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java +++ b/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java @@ -129,6 +129,7 @@ import com.cloud.network.dao.VpnUserDao; import com.cloud.network.lb.LoadBalancingRule; import com.cloud.network.lb.LoadBalancingRule.LbDestination; import com.cloud.network.lb.LoadBalancingRulesManager; +import com.cloud.network.ovs.GreTunnelException; import com.cloud.network.ovs.OvsNetworkManager; import com.cloud.network.router.VirtualRouter.Role; import com.cloud.network.rules.FirewallRule; @@ -1248,8 +1249,13 @@ public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplian @Override public boolean finalizeDeployment(Commands cmds, VirtualMachineProfile profile, DeployDestination dest, ReservationContext context) throws ResourceUnavailableException{ NicProfile controlNic = (NicProfile) profile.getParameter("control.nic"); - _ovsNetworkMgr.RouterCheckAndCreateTunnel(cmds, profile, dest); - _ovsNetworkMgr.applyDefaultFlowToRouter(cmds, profile, dest); + try { + _ovsNetworkMgr.RouterCheckAndCreateTunnel(cmds, profile, dest); + _ovsNetworkMgr.applyDefaultFlowToRouter(cmds, profile, dest); + } catch (GreTunnelException e) { + e.printStackTrace(); + } + cmds.addCommand("checkSsh", new CheckSshCommand(profile.getInstanceName(), controlNic.getIp4Address(), 3922, 5, 20)); diff --git a/server/src/com/cloud/vm/UserVmManagerImpl.java b/server/src/com/cloud/vm/UserVmManagerImpl.java index db71b46f8a7..913f7b0dbe5 100755 --- a/server/src/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/com/cloud/vm/UserVmManagerImpl.java @@ -129,6 +129,7 @@ import com.cloud.network.dao.LoadBalancerDao; import com.cloud.network.dao.LoadBalancerVMMapDao; import com.cloud.network.dao.NetworkDao; import com.cloud.network.lb.LoadBalancingRulesManager; +import com.cloud.network.ovs.GreTunnelException; import com.cloud.network.ovs.OvsNetworkManager; import com.cloud.network.router.VirtualNetworkApplianceManager; import com.cloud.network.rules.RulesManager; @@ -2392,8 +2393,14 @@ public class UserVmManagerImpl implements UserVmManager, UserVmService, Manager } _vmDao.update(userVm.getId(), userVm); - _ovsNetworkMgr.UserVmCheckAndCreateTunnel(cmds, profile, dest); - _ovsNetworkMgr.applyDefaultFlowToUserVm(cmds, profile, dest); + + try { + _ovsNetworkMgr.UserVmCheckAndCreateTunnel(cmds, profile, dest); + _ovsNetworkMgr.applyDefaultFlowToUserVm(cmds, profile, dest); + } catch (GreTunnelException e) { + e.printStackTrace(); + } + return true; } diff --git a/setup/db/create-schema.sql b/setup/db/create-schema.sql index 60a55dd2c45..afa3db648ff 100755 --- a/setup/db/create-schema.sql +++ b/setup/db/create-schema.sql @@ -1332,6 +1332,14 @@ CREATE TABLE `cloud`.`ovs_host_vlan_alloc`( PRIMARY KEY(`id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8; +CREATE TABLE `cloud`.`ovs_tunnel_alloc`( + `id` bigint unsigned NOT NULL UNIQUE AUTO_INCREMENT, + `from` bigint unsigned COMMENT 'from host id', + `to` bigint unsigned COMMENT 'to host id', + `in_port` int unsigned COMMENT 'in port on open vswitch', + PRIMARY KEY(`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + CREATE TABLE `cloud`.`ovs_vlan_mapping_dirty`( `id` bigint unsigned NOT NULL UNIQUE AUTO_INCREMENT, `account_id` bigint unsigned COMMENT 'account id',