CLOUDSTACK-10013: SystemVM codebase refactorings and improvements

- Refactors and simplifies systemvm codebase file structures keeping
  the same resultant systemvm.iso packaging
- Password server systemd script and new postinit script that runs
  before sshd starts
- Fixes to keepalived and conntrackd config to make rVRs work again
- New /etc/issue featuring ascii based cloudmonkey logo/message and
  systemvmtemplate version
- SystemVM python codebase linted and tested. Added pylint/pep to
  Travis.
- iptables re-application fixes for non-VR systemvms.
- SystemVM template build fixes.
- Default secondary storage vm service offering boosted to have 2vCPUs
  and RAM equal to console proxy.
- Fixes to several marvin based smoke tests, especially rVR related
  tests. rVR tests to consider 3*advert_int+skew timeout before status
  is checked.

Signed-off-by: Rohit Yadav <rohit.yadav@shapeblue.com>

.travis.yml

@@ -50,7 +50,6 @@ env:
              smoke/test_hostha_simulator
              smoke/test_host_annotations
              smoke/test_host_maintenance
-             smoke/test_hosts
              smoke/test_internal_lb
              smoke/test_iso
              smoke/test_list_ids_parameter

LICENSE

@@ -208,7 +208,7 @@ Within the console-proxy/js directory
             jquery.js 
 
 
-Within the patches/systemvm/debian/config/etc directory
+Within the systemvm/debian/etc directory
     placed in the public domain
         by Adiscon GmbH  http://www.adiscon.com/ 
             rsyslog.conf 
@@ -216,14 +216,14 @@ Within the patches/systemvm/debian/config/etc directory
             dnsmasq.conf 
             vpcdnsmasq.conf 
 
-Within the patches/systemvm/debian/config/etc/apache2 directory
+Within the systemvm/debian/etc/apache2 directory
     licensed under the Apache License, Version 2 http://www.apache.org/licenses/LICENSE-2.0.txt  (as above)
     Copyright (c) 2012 The Apache Software Foundation
         from The Apache Software Foundation  http://www.apache.org/ 
             httpd.conf 
             vhost.template
 
-Within the patches/systemvm/debian/config/etc/ssh/ directory
+Within the systemvm/debian/etc/ssh/ directory
     licensed under the BSD (2-clause) http://www.opensource.org/licenses/BSD-2-Clause  (as follows)
  
             
@@ -254,7 +254,7 @@ Within the patches/systemvm/debian/config/etc/ssh/ directory
         from OpenSSH Project  http://www.openssh.org/ 
             sshd_config 
 
-Within the patches/systemvm/debian/config/root/redundant_router directory
+Within the systemvm/debian/root/redundant_router directory
     placed in the public domain
         by The netfilter.org project  http://www.netfilter.org/ 
             conntrackd.conf.templ 

agent/src/com/cloud/agent/resource/consoleproxy/ConsoleProxyResource.java

@@ -440,11 +440,6 @@ public class ConsoleProxyResource extends ServerResourceBase implements ServerRe
         }
     }
 
-    @Override
-    public boolean stop() {
-        return true;
-    }
-
     @Override
     public void setName(String name) {
     }

engine/schema/resources/META-INF/db/schema-41000to41100.sql

@@ -492,3 +492,6 @@ INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervis
 
 -- Change monitor patch for apache2 in systemvm
 UPDATE `cloud`.`monitoring_services` SET pidfile="/var/run/apache2/apache2.pid" WHERE process_name="apache2" AND service_name="apache2";
+
+-- Boost secondary storage systemvm
+UPDATE `cloud`.`service_offering` SET ram_size=1024, cpu=2 WHERE vm_type="secondarystoragevm" and cpu=1 and ram_size=512;

plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java

@@ -341,17 +341,18 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
     @Override
     public ExecutionResult createFileInVR(final String routerIp, final String path, final String filename, final String content) {
         final File permKey = new File("/root/.ssh/id_rsa.cloud");
-        String error = null;
+        boolean success = true;
-
+        String details = "Creating file in VR, with ip: " + routerIp + ", file: " + filename;
-        s_logger.debug("Creating file in VR, with ip: " + routerIp + ", file: " + filename);
+        s_logger.debug(details);
 
         try {
             SshHelper.scpTo(routerIp, 3922, "root", permKey, null, path, content.getBytes(), filename, null);
         } catch (final Exception e) {
             s_logger.warn("Fail to create file " + path + filename + " in VR " + routerIp, e);
-            error = e.getMessage();
+            details = e.getMessage();
+            success = false;
         }
-        return new ExecutionResult(error == null, error);
+        return new ExecutionResult(success, details);
     }
 
     @Override

pom.xml

@@ -924,8 +924,7 @@
               <exclude>systemvm/agent/conf/environment.properties</exclude>
               <exclude>systemvm/agent/js/jquery.js</exclude>
               <exclude>systemvm/agent/js/jquery.flot.navigate.js</exclude>
-              <exclude>systemvm/patches/debian/**</exclude>
+              <exclude>systemvm/debian/**</exclude>
-              <exclude>systemvm/patches/vpn/**</exclude>
               <exclude>tools/transifex/.tx/config</exclude>
               <exclude>tools/logo/apache_cloudstack.png</exclude>
               <exclude>tools/marvin/marvin/sandbox/advanced/sandbox.cfg</exclude>

systemvm/debian/etc/issue

@@ -0,0 +1,3 @@
+   __?.o/  Apache CloudStack SystemVM 4.11
+  (  )#    https://cloudstack.apache.org
+ (___(_)   \s \r \n \l

systemvm/debian/etc/rc.local

@@ -0,0 +1,47 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[ ! -f /var/cache/cloud/enabled_svcs ] && touch /var/cache/cloud/enabled_svcs
+for svc in $(cat /var/cache/cloud/enabled_svcs)
+do
+   logger -t cloud "Starting $svc"
+   systemctl enable --no-block --now $svc
+done
+
+[ ! -f /var/cache/cloud/disabled_svcs ] && touch /var/cache/cloud/disabled_svcs
+for svc in $(cat /var/cache/cloud/disabled_svcs)
+do
+   logger -t cloud "Stopping $svc"
+   systemctl disable --no-block --now $svc
+done
+
+# Restore the persistent iptables nat, rules and filters for IPv4 and IPv6 if they exist
+ipv4="/etc/iptables/rules.v4"
+if [ -e $ipv4 ]
+then
+   iptables-restore < $ipv4
+fi
+
+ipv6="/etc/iptables/rules.v6"
+if [ -e $ipv6 ]
+then
+   iptables-restore < $ipv6
+fi
+
+date > /var/cache/cloud/boot_up_done
+logger -t cloud "Boot up process done"

systemvm/debian/etc/ssh/sshd_config

@@ -57,7 +57,6 @@ AuthorizedKeysFile .ssh/authorized_keys
 #IgnoreRhosts yes
 
 # To disable tunneled clear text passwords, change to no here!
-#PasswordAuthentication yes
 #PermitEmptyPasswords no
 PasswordAuthentication no
 

systemvm/debian/etc/systemd/system/cloud-early-config.service

@@ -0,0 +1,18 @@
+[Unit]
+Description=CloudStack post-boot patching service using cmdline
+DefaultDependencies=no
+
+Before=network-pre.target
+Wants=network-pre.target
+
+Requires=local-fs.target
+After=local-fs.target
+
+[Install]
+WantedBy=multi-user.target
+
+[Service]
+Type=oneshot
+ExecStart=/opt/cloud/bin/setup/cloud-early-config
+RemainAfterExit=true
+TimeoutStartSec=5min

systemvm/debian/etc/systemd/system/cloud-password-server@.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=Cloud password server on %I
+After=network.target local-fs.target
+
+[Install]
+WantedBy=multi-user.target
+
+[Service]
+Type=simple
+WorkingDirectory=/opt/cloud/bin
+ExecStart=/opt/cloud/bin/passwd_server_ip.py %I
+Restart=on-failure

systemvm/debian/etc/systemd/system/cloud-postinit.service

@@ -0,0 +1,14 @@
+[Unit]
+Description=CloudStack post-patching init script
+After=cloud-early-config.service network.target local-fs.target
+Before=ssh.service
+Requires=network.service
+
+[Install]
+WantedBy=multi-user.target
+
+[Service]
+Type=oneshot
+ExecStart=/opt/cloud/bin/setup/postinit.sh
+RemainAfterExit=true
+TimeoutStartSec=1min

systemvm/debian/etc/systemd/system/cloud.service

@@ -0,0 +1,13 @@
+[Unit]
+Description=CloudStack Agent service
+After=cloud-early-config.service network.target local-fs.target
+
+[Install]
+WantedBy=multi-user.target
+
+[Service]
+Type=simple
+WorkingDirectory=/usr/local/cloud/systemvm
+ExecStart=/usr/local/cloud/systemvm/_run.sh
+Restart=always
+RestartSec=5

systemvm/debian/opt/cloud/bin/baremetal-vr.py

@@ -15,8 +15,6 @@
 #specific language governing permissions and limitations
 #under the License.
 
-__author__ = 'frank'
-
 import subprocess
 import urllib
 import hmac

systemvm/debian/opt/cloud/bin/configure.py

@@ -16,16 +16,14 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-import sys
 import base64
+import logging
+import os
+import re
+import sys
+import time
 
 from collections import OrderedDict
-
-import logging
-import re
-
-import os.path
-import os
 from fcntl import flock, LOCK_EX, LOCK_UN
 
 from cs.CsDatabag import CsDataBag
@@ -39,11 +37,10 @@ from cs.CsConfig import CsConfig
 from cs.CsProcess import CsProcess
 from cs.CsStaticRoutes import CsStaticRoutes
 
-OCCURRENCES = 1
 
 class CsPassword(CsDataBag):
 
-    TOKEN_FILE="/tmp/passwdsrvrtoken"
+    TOKEN_FILE = "/tmp/passwdsrvrtoken"
 
     def process(self):
         for item in self.dbag:
@@ -66,7 +63,7 @@ class CsPassword(CsDataBag):
             proc = CsProcess(['/opt/cloud/bin/passwd_server_ip.py', server_ip])
             if proc.find():
                 update_command = 'curl --header "DomU_Request: save_password" "http://{SERVER_IP}:8080/" -F "ip={VM_IP}" -F "password={PASSWORD}" ' \
-                '-F "token={TOKEN}" >/dev/null 2>/dev/null &'.format(SERVER_IP=server_ip, VM_IP=vm_ip, PASSWORD=password, TOKEN=token)
+                                 '-F "token={TOKEN}" >/dev/null 2>/dev/null &'.format(SERVER_IP=server_ip, VM_IP=vm_ip, PASSWORD=password, TOKEN=token)
                 result = CsHelper.execute(update_command)
                 logging.debug("Update password server result ==> %s" % result)
 
@@ -96,7 +93,7 @@ class CsAcl(CsDataBag):
             self.rule['allowed'] = True
             self.rule['action'] = "ACCEPT"
 
-            if self.rule['type'] == 'all' and  obj['source_cidr_list']:
+            if self.rule['type'] == 'all' and obj['source_cidr_list']:
                 self.rule['cidr'] = []
             else:
                 self.rule['cidr'] = obj['source_cidr_list']
@@ -127,7 +124,7 @@ class CsAcl(CsDataBag):
             rnge = ''
             if "first_port" in self.rule.keys() and \
                self.rule['first_port'] == self.rule['last_port']:
-                    rnge = " --dport %s " %self.rule['first_port']
+                    rnge = " --dport %s " % self.rule['first_port']
             if "first_port" in self.rule.keys() and \
                self.rule['first_port'] != self.rule['last_port']:
                     rnge = " --dport %s:%s" % (rule['first_port'], rule['last_port'])
@@ -149,21 +146,21 @@ class CsAcl(CsDataBag):
                                         " -p %s " % rule['protocol'] +
                                         "  %s -j %s" % (rnge, self.rule['action'])])
 
-            sflag=False
+            sflag = False
-            dflag=False
+            dflag = False
             if self.direction == 'egress':
                 ruleId = self.rule['id']
-                sourceIpsetName = 'sourceCidrIpset-%d' %ruleId
+                sourceIpsetName = 'sourceCidrIpset-%d' % ruleId
-                destIpsetName = 'destCidrIpset-%d' %ruleId
+                destIpsetName = 'destCidrIpset-%d' % ruleId
 
-                #create source cidr ipset
+                # Create source cidr ipset
                 srcIpset = 'ipset create '+sourceIpsetName + ' hash:net '
                 dstIpset = 'ipset create '+destIpsetName + ' hash:net '
 
                 CsHelper.execute(srcIpset)
                 CsHelper.execute(dstIpset)
                 for cidr in self.rule['cidr']:
-                    ipsetAddCmd = 'ipset add '+ sourceIpsetName + ' '+cidr
+                    ipsetAddCmd = 'ipset add ' + sourceIpsetName + ' ' + cidr
                     CsHelper.execute(ipsetAddCmd)
                     sflag = True
 
@@ -171,7 +168,7 @@ class CsAcl(CsDataBag):
                 for cidr in self.rule['dcidr']:
                     if cidr == CIDR_ALL:
                         continue
-                    ipsetAddCmd = 'ipset add '+ destIpsetName + ' '+cidr
+                    ipsetAddCmd = 'ipset add ' + destIpsetName + ' ' + cidr
                     CsHelper.execute(ipsetAddCmd)
                     dflag = True
 
@@ -196,13 +193,13 @@ class CsAcl(CsDataBag):
                     else:
                         self.rule['action'] = "ACCEPT"
 
-                egressIpsetStr=''
+                egressIpsetStr = ''
-                if sflag == True and dflag == True:
+                if sflag and dflag:
                     egressIpsetStr = " -m set --match-set %s src " % sourceIpsetName + \
                                 " -m set --match-set %s dst " % destIpsetName
-                elif sflag == True:
+                elif sflag:
                     egressIpsetStr = " -m set --match-set %s src " % sourceIpsetName
-                elif dflag == True:
+                elif dflag:
                     egressIpsetStr = " -m set --match-set %s dst " % destIpsetName
 
                 if rule['protocol'] == "icmp":
@@ -359,7 +356,7 @@ class CsVmMetadata(CsDataBag):
             if data is not None:
                 # need to pad data if it is not valid base 64
                 if len(data) % 4 != 0:
-                    data  += (4-(len(data) % 4)) * "="
+                    data += (4 - (len(data) % 4)) * "="
                 data = base64.b64decode(data)
 
         fh = open(dest, "w")
@@ -466,7 +463,7 @@ class CsVmMetadata(CsDataBag):
     def __unflock(self, file):
         try:
             flock(file, LOCK_UN)
-        except IOError:
+        except IOError as e:
             print "failed to unlock file" + file.name + " due to : " + e.strerror
             sys.exit(1)  # FIXME
         return True
@@ -539,8 +536,8 @@ class CsSite2SiteVpn(CsDataBag):
         peerlist = obj['peer_guest_cidr_list'].replace(' ', '')
         vpnconffile = "%s/ipsec.vpn-%s.conf" % (self.VPNCONFDIR, rightpeer)
         vpnsecretsfile = "%s/ipsec.vpn-%s.secrets" % (self.VPNCONFDIR, rightpeer)
-        ikepolicy=obj['ike_policy'].replace(';','-')
+        ikepolicy = obj['ike_policy'].replace(';', '-')
-        esppolicy=obj['esp_policy'].replace(';','-')
+        esppolicy = obj['esp_policy'].replace(';', '-')
 
         if rightpeer in self.confips:
             self.confips.remove(rightpeer)
@@ -561,7 +558,7 @@ class CsSite2SiteVpn(CsDataBag):
         file.addeq(" keyingtries=2")
         file.addeq(" auto=route")
         if 'encap' not in obj:
-            obj['encap']=False
+            obj['encap'] = False
         file.addeq(" forceencaps=%s" % CsHelper.bool_to_yn(obj['encap']))
         if obj['dpd']:
             file.addeq(" dpddelay=30")
@@ -595,14 +592,14 @@ class CsSite2SiteVpn(CsDataBag):
 
 
 class CsVpnUser(CsDataBag):
-    PPP_CHAP='/etc/ppp/chap-secrets'
+    PPP_CHAP = '/etc/ppp/chap-secrets'
 
     def process(self):
         for user in self.dbag:
             if user == 'id':
                 continue
 
-            userconfig=self.dbag[user]
+            userconfig = self.dbag[user]
             if userconfig['add']:
                 self.add_l2tp_ipsec_user(user, userconfig)
             else:
@@ -612,7 +609,7 @@ class CsVpnUser(CsDataBag):
         userfound = False
         password = obj['password']
 
-        userAddEntry = "%s * %s *" %(user,password)
+        userAddEntry = "%s * %s *" % (user, password)
         logging.debug("Adding vpn user '%s'" % user)
 
         file = CsFile(self.PPP_CHAP)
@@ -626,7 +623,7 @@ class CsVpnUser(CsDataBag):
     def del_l2tp_ipsec_user(self, user, obj):
         userfound = False
         password = obj['password']
-        userentry = "%s * %s *" % (user,password)
+        userentry = "%s * %s *" % (user, password)
 
         logging.debug("Deleting the user '%s'" % user)
         file = CsFile(self.PPP_CHAP)
@@ -647,7 +644,7 @@ class CsVpnUser(CsDataBag):
                     if pppd == 'PPPD_PID':
                         pid = str.split('=')[1]
                         if pid:
-                            logging.debug("killing process %s" %pid)
+                            logging.debug("killing process %s" % pid)
                             CsHelper.execute('kill -9 %s' % pid)
 
 
@@ -662,12 +659,11 @@ class CsRemoteAccessVpn(CsDataBag):
         for public_ip in self.dbag:
             if public_ip == "id":
                 continue
-            vpnconfig=self.dbag[public_ip]
+            vpnconfig = self.dbag[public_ip]
 
             # Enable remote access vpn
             if vpnconfig['create']:
-                shutdownIpsec = False
+                logging.debug("Enabling remote access vpn on " + public_ip)
-                logging.debug("Enabling  remote access vpn  on "+ public_ip)
 
                 CsHelper.start_if_stopped("ipsec")
                 self.configure_l2tpIpsec(public_ip, self.dbag[public_ip])
@@ -682,20 +678,19 @@ class CsRemoteAccessVpn(CsDataBag):
                 CsHelper.execute("ipsec down L2TP-PSK")
                 CsHelper.execute("systemctl stop xl2tpd")
 
+    def configure_l2tpIpsec(self, left, obj):
+        l2tpconffile = "%s/l2tp.conf" % (self.VPNCONFDIR)
+        vpnsecretfilte = "%s/ipsec.any.secrets" % (self.VPNCONFDIR)
+        xl2tpdconffile = "/etc/xl2tpd/xl2tpd.conf"
+        xl2tpoptionsfile = "/etc/ppp/options.xl2tpd"
 
-    def configure_l2tpIpsec(self, left,  obj):
+        localip = obj['local_ip']
-        l2tpconffile="%s/l2tp.conf" % (self.VPNCONFDIR)
+        localcidr = obj['local_cidr']
-        vpnsecretfilte="%s/ipsec.any.secrets" % (self.VPNCONFDIR)
+        publicIface = obj['public_interface']
-        xl2tpdconffile="/etc/xl2tpd/xl2tpd.conf"
+        iprange = obj['ip_range']
-        xl2tpoptionsfile='/etc/ppp/options.xl2tpd'
+        psk = obj['preshared_key']
 
-        localip=obj['local_ip']
+        # Left
-        localcidr=obj['local_cidr']
-        publicIface=obj['public_interface']
-        iprange=obj['ip_range']
-        psk=obj['preshared_key']
-
-        #left
         l2tpfile = CsFile(l2tpconffile)
         l2tpfile.addeq(" left=%s" % left)
         l2tpfile.commit()
@@ -706,19 +701,18 @@ class CsRemoteAccessVpn(CsDataBag):
         secret.commit()
 
         xl2tpdconf = CsFile(xl2tpdconffile)
-        xl2tpdconf.addeq("ip range = %s" %iprange)
+        xl2tpdconf.addeq("ip range = %s" % iprange)
-        xl2tpdconf.addeq("local ip = %s" %localip)
+        xl2tpdconf.addeq("local ip = %s" % localip)
         xl2tpdconf.commit()
 
-        xl2tpoptions=CsFile(xl2tpoptionsfile)
+        xl2tpoptions = CsFile(xl2tpoptionsfile)
-        xl2tpoptions.search("ms-dns ", "ms-dns %s" %localip)
+        xl2tpoptions.search("ms-dns ", "ms-dns %s" % localip)
         xl2tpoptions.commit()
 
     def remoteaccessvpn_iptables(self, publicip, obj):
-        publicdev=obj['public_interface']
+        publicdev = obj['public_interface']
-        localcidr=obj['local_cidr']
+        localcidr = obj['local_cidr']
-        local_ip=obj['local_ip']
+        local_ip = obj['local_ip']
-
 
         self.fw.append(["", "", "-A INPUT -i %s --dst %s -p udp -m udp --dport 500 -j ACCEPT" % (publicdev, publicip)])
         self.fw.append(["", "", "-A INPUT -i %s --dst %s -p udp -m udp --dport 4500 -j ACCEPT" % (publicdev, publicip)])
@@ -729,31 +723,30 @@ class CsRemoteAccessVpn(CsDataBag):
         self.fw.append(["", "", "-A OUTPUT -p esp -j ACCEPT"])
 
         if self.config.is_vpc():
-            self.fw.append(["", ""," -N VPN_FORWARD"])
+            self.fw.append(["", "", " -N VPN_FORWARD"])
-            self.fw.append(["", "","-I FORWARD -i ppp+ -j VPN_FORWARD"])
+            self.fw.append(["", "", "-I FORWARD -i ppp+ -j VPN_FORWARD"])
-            self.fw.append(["", "","-I FORWARD -o ppp+ -j VPN_FORWARD"])
+            self.fw.append(["", "", "-I FORWARD -o ppp+ -j VPN_FORWARD"])
-            self.fw.append(["", "","-I FORWARD -o ppp+ -j VPN_FORWARD"])
+            self.fw.append(["", "", "-I FORWARD -o ppp+ -j VPN_FORWARD"])
-            self.fw.append(["", "","-A VPN_FORWARD -s  %s -j RETURN" %localcidr])
+            self.fw.append(["", "", "-A VPN_FORWARD -s  %s -j RETURN" % localcidr])
-            self.fw.append(["", "","-A VPN_FORWARD -i ppp+ -d %s -j RETURN" %localcidr])
+            self.fw.append(["", "", "-A VPN_FORWARD -i ppp+ -d %s -j RETURN" % localcidr])
-            self.fw.append(["", "","-A VPN_FORWARD -i ppp+  -o ppp+ -j RETURN"])
+            self.fw.append(["", "", "-A VPN_FORWARD -i ppp+  -o ppp+ -j RETURN"])
         else:
-            self.fw.append(["", "","-A FORWARD -i ppp+ -o  ppp+ -j ACCEPT"])
+            self.fw.append(["", "", "-A FORWARD -i ppp+ -o  ppp+ -j ACCEPT"])
-            self.fw.append(["", "","-A FORWARD -s %s -o  ppp+ -j ACCEPT" % localcidr])
+            self.fw.append(["", "", "-A FORWARD -s %s -o  ppp+ -j ACCEPT" % localcidr])
-            self.fw.append(["", "","-A FORWARD -i ppp+ -d %s  -j ACCEPT" % localcidr])
+            self.fw.append(["", "", "-A FORWARD -i ppp+ -d %s  -j ACCEPT" % localcidr])
 
-
+        self.fw.append(["", "", "-A INPUT -i ppp+ -m udp -p udp --dport 53 -j ACCEPT"])
-        self.fw.append(["", "","-A INPUT -i ppp+ -m udp -p udp --dport 53 -j ACCEPT"])
+        self.fw.append(["", "", "-A INPUT -i ppp+ -m tcp -p tcp --dport 53 -j ACCEPT"])
-        self.fw.append(["", "","-A INPUT -i ppp+ -m tcp -p tcp --dport 53 -j ACCEPT"])
+        self.fw.append(["nat", "", "-I PREROUTING -i ppp+ -m tcp --dport 53 -j DNAT --to-destination %s" % local_ip])
-        self.fw.append(["nat", "","-I PREROUTING -i ppp+ -m tcp --dport 53 -j DNAT --to-destination %s" % local_ip])
 
         if self.config.is_vpc():
             return
 
-        self.fw.append(["mangle", "","-N  VPN_%s " %publicip])
+        self.fw.append(["mangle", "", "-N  VPN_%s " % publicip])
-        self.fw.append(["mangle", "","-A VPN_%s -j RETURN " % publicip])
+        self.fw.append(["mangle", "", "-A VPN_%s -j RETURN " % publicip])
-        self.fw.append(["mangle", "","-I VPN_%s -p ah  -j ACCEPT " % publicip])
+        self.fw.append(["mangle", "", "-I VPN_%s -p ah  -j ACCEPT " % publicip])
-        self.fw.append(["mangle", "","-I VPN_%s -p esp  -j ACCEPT " % publicip])
+        self.fw.append(["mangle", "", "-I VPN_%s -p esp  -j ACCEPT " % publicip])
-        self.fw.append(["mangle", "","-I PREROUTING  -d %s -j VPN_%s " % (publicip, publicip)])
+        self.fw.append(["mangle", "", "-I PREROUTING  -d %s -j VPN_%s " % (publicip, publicip)])
 
 
 class CsForwardingRules(CsDataBag):
@@ -768,7 +761,7 @@ class CsForwardingRules(CsDataBag):
                 elif rule["type"] == "staticnat":
                     self.processStaticNatRule(rule)
 
-    #return the VR guest interface ip
+    # Return the VR guest interface ip
     def getGuestIp(self):
         interfaces = []
         ipAddr = None
@@ -814,7 +807,7 @@ class CsForwardingRules(CsDataBag):
             self.forward_vr(rule)
 
     def forward_vr(self, rule):
-        #prefetch iptables variables
+        # Prefetch iptables variables
         public_fwinterface = self.getDeviceByIp(rule['public_ip'])
         internal_fwinterface = self.getDeviceByIp(rule['internal_ip'])
         public_fwports = self.portsToString(rule['public_ports'], ':')
@@ -930,10 +923,10 @@ class CsForwardingRules(CsDataBag):
             raise Exception("Ip address %s has no device in the ips databag" % rule["public_ip"])
 
         self.fw.append(["mangle", "",
-                        "-I PREROUTING -s %s/32 -m state --state NEW -j CONNMARK --save-mark --nfmask 0xffffffff --ctmask 0xffffffff" % \
+                        "-I PREROUTING -s %s/32 -m state --state NEW -j CONNMARK --save-mark --nfmask 0xffffffff --ctmask 0xffffffff" %
                         rule["internal_ip"]])
         self.fw.append(["mangle", "",
-                        "-I PREROUTING -s %s/32 -m state --state NEW -j MARK --set-xmark 0x%s/0xffffffff" % \
+                        "-I PREROUTING -s %s/32 -m state --state NEW -j MARK --set-xmark 0x%s/0xffffffff" %
                         (rule["internal_ip"], device[len("eth"):])])
         self.fw.append(["nat", "front",
                         "-A PREROUTING -d %s/32 -j DNAT --to-destination %s" % (rule["public_ip"], rule["internal_ip"])])
@@ -944,11 +937,12 @@ class CsForwardingRules(CsDataBag):
         self.fw.append(["filter", "",
                         "-A FORWARD -i %s -o eth0  -d %s  -m state  --state NEW -j ACCEPT " % (device, rule["internal_ip"])])
 
-        #configure the hairpin nat
+        # Configure the hairpin nat
         self.fw.append(["nat", "front",
                         "-A PREROUTING -d %s -i eth0 -j DNAT --to-destination %s" % (rule["public_ip"], rule["internal_ip"])])
 
-        self.fw.append(["nat", "front", "-A POSTROUTING -s %s -d %s -j SNAT -o eth0 --to-source %s" % (self.getNetworkByIp(rule['internal_ip']),rule["internal_ip"], self.getGuestIp())])
+        self.fw.append(["nat", "front", "-A POSTROUTING -s %s -d %s -j SNAT -o eth0 --to-source %s" %
+                        (self.getNetworkByIp(rule['internal_ip']), rule["internal_ip"], self.getGuestIp())])
 
 
 class IpTablesExecutor:
@@ -984,8 +978,9 @@ class IpTablesExecutor:
         logging.debug("Configuring iptables rules done ...saving rules")
 
         # Save iptables configuration - will be loaded on reboot by the iptables-restore that is configured on /etc/rc.local
-        CsHelper.save_iptables("iptables-save", "/etc/iptables/router_rules.v4")
+        CsHelper.save_iptables("iptables-save", "/etc/iptables/rules.v4")
-        CsHelper.save_iptables("ip6tables-save", "/etc/iptables/router_rules.v6")
+        CsHelper.save_iptables("ip6tables-save", "/etc/iptables/rules.v6")
+
 
 def main(argv):
     # The file we are currently processing, if it is "cmd_line.json" everything will be processed.
@@ -995,6 +990,8 @@ def main(argv):
         logging.debug("No file was received, do not go on processing the other actions. Just leave for now.")
         return
 
+    json_type = os.path.basename(process_file).split('.json')[0]
+
     # The "GLOBAL" Configuration object
     config = CsConfig()
 
@@ -1009,53 +1006,51 @@ def main(argv):
     config.address().compare()
     config.address().process()
 
-    databag_map = OrderedDict([("guest_network.json", {"process_iptables" : True, "executor" : IpTablesExecutor(config)}),
+    databag_map = OrderedDict([("guest_network",     {"process_iptables": True,  "executor": []}),
-                               ("vm_password.json", {"process_iptables" : False, "executor" : CsPassword("vmpassword", config)}),
+                               ("vm_password",       {"process_iptables": False, "executor": [CsPassword("vmpassword", config)]}),
-                               ("vm_metadata.json", {"process_iptables" : False, "executor" : CsVmMetadata('vmdata', config)}),
+                               ("vm_metadata",       {"process_iptables": False, "executor": [CsVmMetadata('vmdata', config)]}),
-                               ("network_acl.json", {"process_iptables" : True, "executor" : IpTablesExecutor(config)}),
+                               ("network_acl",       {"process_iptables": True,  "executor": []}),
-                               ("firewall_rules.json", {"process_iptables" : True, "executor" : IpTablesExecutor(config)}),
+                               ("firewall_rules",    {"process_iptables": True,  "executor": []}),
-                               ("forwarding_rules.json", {"process_iptables" : True, "executor" : IpTablesExecutor(config)}),
+                               ("forwarding_rules",  {"process_iptables": True,  "executor": []}),
-                               ("staticnat_rules.json", {"process_iptables" : True, "executor" : IpTablesExecutor(config)}),
+                               ("staticnat_rules",   {"process_iptables": True,  "executor": []}),
-                               ("site_2_site_vpn.json", {"process_iptables" : True, "executor" : IpTablesExecutor(config)}),
+                               ("site_2_site_vpn",   {"process_iptables": True,  "executor": []}),
-                               ("remote_access_vpn.json", {"process_iptables" : True, "executor" : IpTablesExecutor(config)}),
+                               ("remote_access_vpn", {"process_iptables": True,  "executor": []}),
-                               ("vpn_user_list.json", {"process_iptables" : False, "executor" : CsVpnUser("vpnuserlist", config)}),
+                               ("vpn_user_list",     {"process_iptables": False, "executor": [CsVpnUser("vpnuserlist", config)]}),
-                               ("vm_dhcp_entry.json", {"process_iptables" : False, "executor" : CsDhcp("dhcpentry", config)}),
+                               ("vm_dhcp_entry",     {"process_iptables": False, "executor": [CsDhcp("dhcpentry", config)]}),
-                               ("dhcp.json", {"process_iptables" : False, "executor" : CsDhcp("dhcpentry", config)}),
+                               ("dhcp",              {"process_iptables": False, "executor": [CsDhcp("dhcpentry", config)]}),
-                               ("load_balancer.json", {"process_iptables" : True, "executor" : IpTablesExecutor(config)}),
+                               ("load_balancer",     {"process_iptables": True,  "executor": []}),
-                               ("monitor_service.json", {"process_iptables" : False, "executor" : CsMonitor("monitorservice", config)}),
+                               ("monitor_service",   {"process_iptables": False, "executor": [CsMonitor("monitorservice", config)]}),
-                               ("static_routes.json", {"process_iptables" : False, "executor" : CsStaticRoutes("staticroutes", config)})
+                               ("static_routes",     {"process_iptables": False, "executor": [CsStaticRoutes("staticroutes", config)]})
                                ])
 
-    if process_file.count("cmd_line.json") == OCCURRENCES:
+    def execDatabag(key, db):
-        logging.debug("cmd_line.json changed. All other files will be processed as well.")
+        if key not in db.keys() or 'executor' not in db[key]:
-
+            logging.warn("Unable to find config or executor(s) for the databag type %s" % key)
-        while databag_map:
+            return
-            item = databag_map.popitem(last = False)
+        for executor in db[key]['executor']:
-            item_name = item[0]
+            logging.debug("Processing for databag type: %s" % key)
-            item_dict = item[1]
+            executor.process()
-            if not item_dict["process_iptables"]:
-                executor = item_dict["executor"]
-                executor.process()
 
+    def execIptables(config):
+        logging.debug("Processing iptables rules")
         iptables_executor = IpTablesExecutor(config)
         iptables_executor.process()
+
+    if json_type == "cmd_line":
+        logging.debug("cmd_line.json changed. All other files will be processed as well.")
+        for key in databag_map.keys():
+            execDatabag(key, databag_map)
+        execIptables(config)
+    elif json_type in databag_map.keys():
+        execDatabag(json_type, databag_map)
+        if databag_map[json_type]['process_iptables']:
+            execIptables(config)
     else:
-        while databag_map:
+        logging.warn("Unable to find and process databag for file: %s, for json type=%s" % (process_file, json_type))
-            item = databag_map.popitem(last = False)
-            item_name = item[0]
-            item_dict = item[1]
-            if process_file.count(item_name) == OCCURRENCES:
-                executor = item_dict["executor"]
-                executor.process()
-
-                if item_dict["process_iptables"]:
-                    iptables_executor = IpTablesExecutor(config)
-                    iptables_executor.process()
-
-                break
 
     red = CsRedundant(config)
     red.set()
+    return 0
 
 if __name__ == "__main__":
     main(sys.argv)

systemvm/debian/opt/cloud/bin/cs/CsAddress.py

@@ -27,6 +27,7 @@ from CsRule import CsRule
 
 VRRP_TYPES = ['guest']
 
+
 class CsAddress(CsDataBag):
 
     def compare(self):
@@ -359,7 +360,6 @@ class CsIP:
         self.fw.append(["filter", "", "-P INPUT DROP"])
         self.fw.append(["filter", "", "-P FORWARD DROP"])
 
-
     def fw_router(self):
         if self.config.is_vpc():
             return
@@ -439,7 +439,7 @@ class CsIP:
 
         if self.get_type() in ["guest"]:
             self.fw.append(["mangle", "front", "-A PREROUTING " +
-                            " -i %s -m state --state RELATED,ESTABLISHED "  % self.dev +
+                            " -i %s -m state --state RELATED,ESTABLISHED " % self.dev +
                             "-j CONNMARK --restore-mark --nfmask 0xffffffff --ctmask 0xffffffff"])
             guestNetworkCidr = self.address['network']
             self.fw.append(["filter", "", "-A FORWARD -d %s -o %s -j ACL_INBOUND_%s" %

systemvm/debian/opt/cloud/bin/cs/CsApp.py

@@ -16,7 +16,6 @@
 # specific language governing permissions and limitations
 # under the License.
 import os
-import CsHelper
 from CsFile import CsFile
 from CsProcess import CsProcess
 import CsHelper
@@ -74,20 +73,13 @@ class CsPasswdSvc():
         self.ip = ip
 
     def start(self):
-        proc = CsProcess(["dummy"])
+        CsHelper.service("cloud-password-server@%s" % self.ip, "start")
-        if proc.grep("passwd_server_ip %s" % self.ip) == -1:
-            proc.start("/opt/cloud/bin/passwd_server_ip %s >> /var/log/cloud.log 2>&1" % self.ip, "&")
 
     def stop(self):
-        proc = CsProcess(["Password Service"])
+        CsHelper.service("cloud-password-server@%s" % self.ip, "stop")
-        pid = proc.grep("passwd_server_ip %s" % self.ip)
-        proc.kill(pid)
-        pid = proc.grep("8080,reuseaddr,fork,crnl,bind=%s" % self.ip)
-        proc.kill(pid)
 
     def restart(self):
-        self.stop()
+        CsHelper.service("cloud-password-server@%s" % self.ip, "restart")
-        self.start()
 
 
 class CsDnsmasq(CsApp):

systemvm/debian/opt/cloud/bin/cs/CsDatabag.py

@@ -136,8 +136,8 @@ class CsCmdLine(CsDataBag):
         This is slightly difficult to happen, but if it does, destroy the router with the password generated with the
         code below and restart the VPC with out the clean up option.
         '''
-        if(self.get_type()=='router'):
+        if self.get_type() == 'router':
-            passwd="%s-%s" % (self.get_eth2_ip(), self.get_router_id())
+            passwd = "%s-%s" % (self.get_eth2_ip(), self.get_router_id())
         else:
             passwd = "%s-%s" % (self.get_vpccidr(), self.get_router_id())
         md5 = hashlib.md5()

systemvm/debian/opt/cloud/bin/cs/CsDhcp.py

@@ -135,13 +135,13 @@ class CsDhcp(CsDataBag):
         # with a splay of 60 hours to prevent storms
         lease = randint(700, 760)
 
-        if entry['default_entry'] == True:
+        if entry['default_entry']:
             self.cloud.add("%s,%s,%s,%sh" % (entry['mac_address'],
                                              entry['ipv4_address'],
                                              entry['host_name'],
                                              lease))
         else:
-            tag = entry['ipv4_address'].replace(".","_")
+            tag = entry['ipv4_address'].replace(".", "_")
             self.cloud.add("%s,set:%s,%s,%s,%sh" % (entry['mac_address'],
                                                     tag,
                                                     entry['ipv4_address'],

systemvm/debian/opt/cloud/bin/cs/CsFile.py

@@ -67,7 +67,6 @@ class CsFile:
         self.config = list(self.new_config)
         logging.info("Updated file in-cache configuration")
 
-
     def dump(self):
         for line in self.new_config:
             print line
@@ -134,13 +133,12 @@ class CsFile:
             return True
         return False
 
-
     def searchString(self, search, ignoreLinesStartWith):
         found = False
         logging.debug("Searching for %s string " % search)
 
         for index, line in enumerate(self.new_config):
-            print ' line = ' +line
+            print ' line = ' + line
             if line.lstrip().startswith(ignoreLinesStartWith):
                 continue
             if search in line:
@@ -149,7 +147,6 @@ class CsFile:
 
         return found
 
-
     def deleteLine(self, search):
         found = False
         logging.debug("Searching for %s to remove the line " % search)
@@ -162,7 +159,6 @@ class CsFile:
 
         self.new_config = list(temp_config)
 
-
     def compare(self, o):
         result = (isinstance(o, self.__class__) and set(self.config) == set(o.config))
         logging.debug("Comparison of CsFiles content is ==> %s" % result)

systemvm/debian/opt/cloud/bin/cs/CsGuestNetwork.py

@@ -40,7 +40,7 @@ class CsGuestNetwork:
             return self.config.get_dns()
 
         dns = []
-        if not self.config.use_extdns() and 'router_guest_gateway' in self.data:
+        if 'router_guest_gateway' in self.data and not self.config.use_extdns():
             dns.append(self.data['router_guest_gateway'])
 
         if 'dns' in self.data:

systemvm/debian/opt/cloud/bin/cs/CsHelper.py

@@ -21,16 +21,18 @@ for use in the configuration process
 """
 import subprocess
 import logging
+import sys
 import os.path
 import re
 import shutil
 from netaddr import *
 from pprint import pprint
 
-PUBLIC_INTERFACES = {"router" : "eth2", "vpcrouter" : "eth1"}
+PUBLIC_INTERFACES = {"router": "eth2", "vpcrouter": "eth1"}
+
+STATE_COMMANDS = {"router": "ip addr | grep eth0 | grep inet | wc -l | xargs bash -c  'if [ $0 == 2 ]; then echo \"MASTER\"; else echo \"BACKUP\"; fi'",
+                  "vpcrouter": "ip addr | grep eth1 | grep state | awk '{print $9;}' | xargs bash -c 'if [ $0 == \"UP\" ]; then echo \"MASTER\"; else echo \"BACKUP\"; fi'"}
 
-STATE_COMMANDS = {"router" : "ip addr | grep eth0 | grep inet | wc -l | xargs bash -c  'if [ $0 == 2 ]; then echo \"MASTER\"; else echo \"BACKUP\"; fi'",
-                  "vpcrouter" : "ip addr | grep eth1 | grep state | awk '{print $9;}' | xargs bash -c 'if [ $0 == \"UP\" ]; then echo \"MASTER\"; else echo \"BACKUP\"; fi'"}
 
 def reconfigure_interfaces(router_config, interfaces):
     for interface in interfaces:
@@ -52,6 +54,7 @@ def reconfigure_interfaces(router_config, interfaces):
                 else:
                     execute(cmd)
 
+
 def is_mounted(name):
     for i in execute("mount"):
         vals = i.lstrip().split()
@@ -242,6 +245,7 @@ def copy_if_needed(src, dest):
         return
     copy(src, dest)
 
+
 def copy(src, dest):
     """
     copy source to destination.
@@ -249,6 +253,6 @@ def copy(src, dest):
     try:
         shutil.copy2(src, dest)
     except IOError:
-        logging.Error("Could not copy %s to %s" % (src, dest))
+        logging.error("Could not copy %s to %s" % (src, dest))
     else:
         logging.info("Copied %s to %s" % (src, dest))

systemvm/debian/opt/cloud/bin/cs/CsNetfilter.py

@@ -143,7 +143,7 @@ class CsNetfilters(object):
         # PASS 2: Create rules
         for fw in list:
             tupledFw = tuple(fw)
-            if tupledFw in ruleSet :
+            if tupledFw in ruleSet:
                 logging.debug("Already processed : %s", tupledFw)
                 continue
 
@@ -173,12 +173,12 @@ class CsNetfilters(object):
                     else:
                         cpy = cpy.replace("-A %s" % new_rule.get_chain(), '-I %s %s' % (new_rule.get_chain(), fw[1]))
                 ret = CsHelper.execute2("iptables -t %s %s" % (new_rule.get_table(), cpy))
-                #There are some issues in this framework causing failures  .. like adding a chain without checking it is present causing
+                # There are some issues in this framework causing failures  .. like adding a chain without checking it is present causing
                 # the failures. Also some of the rule like removeFromLoadBalancerRule is deleting rule and deleteLoadBalancerRule
-                #trying to delete which causes the failure.
+                # trying to delete which causes the failure.
-                #For now raising the log.
+                # For now raising the log.
-                #TODO: Need to fix in the framework.
+                # TODO: Need to fix in the framework.
-                if ret.returncode != 0 :
+                if ret.returncode != 0:
                     error = ret.communicate()[0]
                     logging.debug("iptables command got failed ... continuing")
                 ruleSet.add(tupledFw)

systemvm/debian/opt/cloud/bin/cs/CsRedundant.py

@@ -42,6 +42,7 @@ from CsStaticRoutes import CsStaticRoutes
 import socket
 from time import sleep
 
+
 class CsRedundant(object):
 
     CS_RAMDISK_DIR = "/ramdisk"
@@ -74,10 +75,6 @@ class CsRedundant(object):
     def _redundant_off(self):
         CsHelper.service("conntrackd", "stop")
         CsHelper.service("keepalived", "stop")
-        #CsHelper.umount_tmpfs(self.CS_RAMDISK_DIR)
-        #CsHelper.rmdir(self.CS_RAMDISK_DIR)
-        CsHelper.rm(self.CONNTRACKD_CONF)
-        CsHelper.rm(self.KEEPALIVED_CONF)
 
     def _redundant_on(self):
         guest = self.address.get_guest_if()
@@ -110,9 +107,10 @@ class CsRedundant(object):
             CsHelper.service("keepalived", "stop")
             return
 
-        #CsHelper.mkdir(self.CS_RAMDISK_DIR, 0755, False)
+        # setup_router should execute this already:
-        #CsHelper.mount_tmpfs(self.CS_RAMDISK_DIR)
+        # CsHelper.mkdir(self.CS_RAMDISK_DIR, 0755, False)
-        #CsHelper.mkdir(self.CS_ROUTER_DIR, 0755, False)
+        # CsHelper.mount_tmpfs(self.CS_RAMDISK_DIR)
+        # CsHelper.mkdir(self.CS_ROUTER_DIR, 0755, False)
         for s in self.CS_TEMPLATES:
             d = s
             if s.endswith(".templ"):
@@ -143,7 +141,7 @@ class CsRedundant(object):
 
         keepalived_conf.greplace("[RROUTER_BIN_PATH]", self.CS_ROUTER_DIR)
         keepalived_conf.section("authentication {", "}", [
-                                "        auth_type AH \n", "        auth_pass %s\n" % self.cl.get_router_password()])
+                                "        auth_type AH \n", "        auth_pass %s\n" % self.cl.get_router_password()[:8]])
         keepalived_conf.section(
             "virtual_ipaddress {", "}", self._collect_ips())
 
@@ -195,6 +193,7 @@ class CsRedundant(object):
         proc = CsProcess(['/usr/sbin/keepalived'])
         if not proc.find() or keepalived_conf.is_changed() or force_keepalived_restart:
             keepalived_conf.commit()
+            os.chmod(self.KEEPALIVED_CONF, 0o644)
             CsHelper.service("keepalived", "restart")
 
     def release_lock(self):
@@ -371,10 +370,10 @@ class CsRedundant(object):
         lines = []
         for interface in self.address.get_interfaces():
             if interface.needs_vrrp():
-                cmdline=self.config.get_cmdline_instance()
+                cmdline = self.config.get_cmdline_instance()
                 if not interface.is_added():
                     continue
-                if(cmdline.get_type()=='router'):
+                if cmdline.get_type() == 'router':
                     str = "        %s brd %s dev %s\n" % (cmdline.get_guest_gw(), interface.get_broadcast(), interface.get_device())
                 else:
                     str = "        %s brd %s dev %s\n" % (interface.get_gateway_cidr(), interface.get_broadcast(), interface.get_device())

systemvm/debian/opt/cloud/bin/cs/CsRoute.py

@@ -34,14 +34,13 @@ class CsRoute:
         tablename = self.get_tablename(devicename)
         str = "%s %s" % (tablenumber, tablename)
         filename = "/etc/iproute2/rt_tables"
-        logging.info(
+        logging.info("Adding route table: " + str + " to " + filename + " if not present ")
-            "Adding route table: " + str + " to " + filename + " if not present ")
         if not CsHelper.definedinfile(filename, str):
-             CsHelper.execute("sudo echo " + str + " >> /etc/iproute2/rt_tables")
+            CsHelper.execute("sudo echo " + str + " >> /etc/iproute2/rt_tables")
         # remove "from all table tablename" if exists, else it will interfer with
         # routing of unintended traffic
         if self.findRule("from all lookup " + tablename):
-             CsHelper.execute("sudo ip rule delete from all table " + tablename)
+            CsHelper.execute("sudo ip rule delete from all table " + tablename)
 
     def flush_table(self, tablename):
         CsHelper.execute("ip route flush table %s" % (tablename))
@@ -115,4 +114,4 @@ class CsRoute:
         for i in CsHelper.execute("ip rule show"):
             if rule in i.strip():
                 return True
-        return False
+        return False

systemvm/debian/opt/cloud/bin/cs/CsRule.py

@@ -49,7 +49,7 @@ class CsRule:
             logging.info("Added fwmark rule for %s" % (self.table))
 
     def delMark(self):
-        if  self.findMark():
+        if self.findMark():
             cmd = "ip rule delete fwmark %s table %s" % (self.tableNo, self.table)
             CsHelper.execute(cmd)
             logging.info("Deleting fwmark rule for %s" % (self.table))

systemvm/debian/opt/cloud/bin/cs_ip.py

@@ -18,6 +18,7 @@
 
 from netaddr import *
 
+
 def merge(dbag, ip):
     nic_dev_id = None
     for dev in dbag:
@@ -31,7 +32,7 @@ def merge(dbag, ip):
 
     ipo = IPNetwork(ip['public_ip'] + '/' + ip['netmask'])
     if 'nic_dev_id' in ip:
-         nic_dev_id = ip['nic_dev_id']
+        nic_dev_id = ip['nic_dev_id']
     ip['device'] = 'eth' + str(nic_dev_id)
     ip['broadcast'] = str(ipo.broadcast)
     ip['cidr'] = str(ipo.ip) + '/' + str(ipo.prefixlen)
@@ -45,7 +46,7 @@ def merge(dbag, ip):
         dbag[ip['device']] = [ip]
     else:
         if 'source_nat' in ip and ip['source_nat'] and ip['device'] in dbag and len(dbag[ip['device']]) > 0:
-            dbag[ip['device']].insert(0, ip) # make sure the source_nat ip is first (primary) on the device
+            dbag[ip['device']].insert(0, ip)  # Make sure the source_nat ip is first (primary) on the device
         else:
             dbag.setdefault(ip['device'], []).append(ip)
 

systemvm/debian/opt/cloud/bin/cs_vmp.py

@@ -18,6 +18,7 @@
 from pprint import pprint
 from netaddr import *
 
+
 def merge(dbag, data):
     """
     create a dictionary of values new in data ignoring those in dbag and return it

systemvm/debian/opt/cloud/bin/cs_vpnusers.py

@@ -38,8 +38,8 @@ def merge(dbag, data):
             del(dbagc[user])
 
     for user in data['vpn_users']:
-        username=user['user']
+        username = user['user']
-        add=user['add']
+        add = user['add']
         if username not in dbagc.keys():
             dbagc[username] = user
         elif username in dbagc.keys() and not add:

systemvm/debian/opt/cloud/bin/master.py

@@ -42,7 +42,7 @@ logging.basicConfig(filename=config.get_logger(),
                     format=config.get_format())
 config.cmdline()
 cl = CsCmdLine("cmdline", config)
-#Update the configuration to set state as backup and let keepalived decide who the real Master is!
+# Update the configuration to set state as backup and let keepalived decide who the real Master is!
 cl.set_master_state(False)
 cl.save()
 

systemvm/debian/opt/cloud/bin/merge.py

@@ -271,6 +271,7 @@ class updateDataBag:
             dbag = cs_ip.merge(dbag, ip)
         return dbag
 
+
 class QueueFile:
 
     fileName = ''
@@ -284,15 +285,15 @@ class QueueFile:
             self.type = self.data["type"]
             updateDataBag(self)
             return
-        filename = '{cache_location}/{json_file}'.format(cache_location = self.configCache, json_file = self.fileName)
+        filename = '{cache_location}/{json_file}'.format(cache_location=self.configCache, json_file=self.fileName)
         try:
             handle = open(filename)
         except IOError as exception:
-            error_message = ("Exception occurred with the following exception error '{error}'. Could not open '{file}'. "
+            error_message = ("Exception occurred with the following exception error '{error}'. Could not open '{filename}'. "
-                             "It seems that the file has already been moved.".format(error = exception, file = filename))
+                             "It seems that the file has already been moved.".format(error=exception, filename=filename))
             logging.error(error_message)
         else:
-            logging.info("Continuing with the processing of file '{file}'".format(file = filename))
+            logging.info("Continuing with the processing of file '{filename}'".format(filename=filename))
 
             self.data = json.load(handle)
             self.type = self.data["type"]
@@ -331,7 +332,6 @@ class QueueFile:
 
 class PrivateGatewayHack:
 
-
     @classmethod
     def update_network_type_for_privategateway(cls, dbag, data):
         ip = data['router_guest_ip'] if 'router_guest_ip' in data.keys() else data['public_ip']
@@ -347,12 +347,10 @@ class PrivateGatewayHack:
             logging.debug("Not updating nw_type for ip %s because has_private_gw_ip = %s and private_gw_matches = %s " % (ip, has_private_gw_ip, private_gw_matches))
         return data
 
-
     @classmethod
     def if_config_has_privategateway(cls, dbag):
         return 'privategateway' in dbag['config'].keys() and dbag['config']['privategateway'] != "None"
 
-
     @classmethod
     def ip_matches_private_gateway_ip(cls, ip, private_gateway_ip):
         new_ip_matches_private_gateway_ip = False
@@ -360,7 +358,6 @@ class PrivateGatewayHack:
             new_ip_matches_private_gateway_ip = True
         return new_ip_matches_private_gateway_ip
 
-
     @classmethod
     def load_inital_data(cls):
         initial_data_bag = DataBag()