diff --git a/.travis.yml b/.travis.yml index 30549d272f1..a206cbeedf1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -50,7 +50,6 @@ env: smoke/test_hostha_simulator smoke/test_host_annotations smoke/test_host_maintenance - smoke/test_hosts smoke/test_internal_lb smoke/test_iso smoke/test_list_ids_parameter diff --git a/LICENSE b/LICENSE index 982b15844f4..c30d8593c8e 100644 --- a/LICENSE +++ b/LICENSE @@ -208,7 +208,7 @@ Within the console-proxy/js directory jquery.js -Within the patches/systemvm/debian/config/etc directory +Within the systemvm/debian/etc directory placed in the public domain by Adiscon GmbH http://www.adiscon.com/ rsyslog.conf @@ -216,14 +216,14 @@ Within the patches/systemvm/debian/config/etc directory dnsmasq.conf vpcdnsmasq.conf -Within the patches/systemvm/debian/config/etc/apache2 directory +Within the systemvm/debian/etc/apache2 directory licensed under the Apache License, Version 2 http://www.apache.org/licenses/LICENSE-2.0.txt (as above) Copyright (c) 2012 The Apache Software Foundation from The Apache Software Foundation http://www.apache.org/ httpd.conf vhost.template -Within the patches/systemvm/debian/config/etc/ssh/ directory +Within the systemvm/debian/etc/ssh/ directory licensed under the BSD (2-clause) http://www.opensource.org/licenses/BSD-2-Clause (as follows) @@ -254,7 +254,7 @@ Within the patches/systemvm/debian/config/etc/ssh/ directory from OpenSSH Project http://www.openssh.org/ sshd_config -Within the patches/systemvm/debian/config/root/redundant_router directory +Within the systemvm/debian/root/redundant_router directory placed in the public domain by The netfilter.org project http://www.netfilter.org/ conntrackd.conf.templ diff --git a/agent/src/com/cloud/agent/resource/consoleproxy/ConsoleProxyResource.java b/agent/src/com/cloud/agent/resource/consoleproxy/ConsoleProxyResource.java index 24e82572109..86d3bebb38b 100644 --- a/agent/src/com/cloud/agent/resource/consoleproxy/ConsoleProxyResource.java +++ b/agent/src/com/cloud/agent/resource/consoleproxy/ConsoleProxyResource.java @@ -440,11 +440,6 @@ public class ConsoleProxyResource extends ServerResourceBase implements ServerRe } } - @Override - public boolean stop() { - return true; - } - @Override public void setName(String name) { } diff --git a/engine/schema/resources/META-INF/db/schema-41000to41100.sql b/engine/schema/resources/META-INF/db/schema-41000to41100.sql index 5f7eff2dd9e..2a2fe2ddede 100644 --- a/engine/schema/resources/META-INF/db/schema-41000to41100.sql +++ b/engine/schema/resources/META-INF/db/schema-41000to41100.sql @@ -492,3 +492,6 @@ INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervis -- Change monitor patch for apache2 in systemvm UPDATE `cloud`.`monitoring_services` SET pidfile="/var/run/apache2/apache2.pid" WHERE process_name="apache2" AND service_name="apache2"; + +-- Boost secondary storage systemvm +UPDATE `cloud`.`service_offering` SET ram_size=1024, cpu=2 WHERE vm_type="secondarystoragevm" and cpu=1 and ram_size=512; diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 7e68ed3bce0..0ead0857492 100644 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -341,17 +341,18 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv @Override public ExecutionResult createFileInVR(final String routerIp, final String path, final String filename, final String content) { final File permKey = new File("/root/.ssh/id_rsa.cloud"); - String error = null; - - s_logger.debug("Creating file in VR, with ip: " + routerIp + ", file: " + filename); + boolean success = true; + String details = "Creating file in VR, with ip: " + routerIp + ", file: " + filename; + s_logger.debug(details); try { SshHelper.scpTo(routerIp, 3922, "root", permKey, null, path, content.getBytes(), filename, null); } catch (final Exception e) { s_logger.warn("Fail to create file " + path + filename + " in VR " + routerIp, e); - error = e.getMessage(); + details = e.getMessage(); + success = false; } - return new ExecutionResult(error == null, error); + return new ExecutionResult(success, details); } @Override diff --git a/pom.xml b/pom.xml index c30247aaf3e..39a611d5e9d 100644 --- a/pom.xml +++ b/pom.xml @@ -924,8 +924,7 @@ systemvm/agent/conf/environment.properties systemvm/agent/js/jquery.js systemvm/agent/js/jquery.flot.navigate.js - systemvm/patches/debian/** - systemvm/patches/vpn/** + systemvm/debian/** tools/transifex/.tx/config tools/logo/apache_cloudstack.png tools/marvin/marvin/sandbox/advanced/sandbox.cfg diff --git a/systemvm/patches/debian/etc/apache2/httpd.conf b/systemvm/debian/etc/apache2/httpd.conf similarity index 100% rename from systemvm/patches/debian/etc/apache2/httpd.conf rename to systemvm/debian/etc/apache2/httpd.conf diff --git a/systemvm/patches/debian/etc/apache2/vhost.template b/systemvm/debian/etc/apache2/vhost.template similarity index 100% rename from systemvm/patches/debian/etc/apache2/vhost.template rename to systemvm/debian/etc/apache2/vhost.template diff --git a/systemvm/patches/debian/etc/cloud-nic.rules b/systemvm/debian/etc/cloud-nic.rules similarity index 100% rename from systemvm/patches/debian/etc/cloud-nic.rules rename to systemvm/debian/etc/cloud-nic.rules diff --git a/systemvm/patches/debian/etc/dnsmasq.conf.tmpl b/systemvm/debian/etc/dnsmasq.conf.tmpl similarity index 100% rename from systemvm/patches/debian/etc/dnsmasq.conf.tmpl rename to systemvm/debian/etc/dnsmasq.conf.tmpl diff --git a/systemvm/patches/debian/etc/haproxy/haproxy.cfg b/systemvm/debian/etc/haproxy/haproxy.cfg similarity index 100% rename from systemvm/patches/debian/etc/haproxy/haproxy.cfg rename to systemvm/debian/etc/haproxy/haproxy.cfg diff --git a/systemvm/patches/vpn/etc/ipsec.conf b/systemvm/debian/etc/ipsec.conf similarity index 100% rename from systemvm/patches/vpn/etc/ipsec.conf rename to systemvm/debian/etc/ipsec.conf diff --git a/systemvm/patches/vpn/etc/ipsec.d/l2tp.conf b/systemvm/debian/etc/ipsec.d/l2tp.conf similarity index 100% rename from systemvm/patches/vpn/etc/ipsec.d/l2tp.conf rename to systemvm/debian/etc/ipsec.d/l2tp.conf diff --git a/systemvm/patches/vpn/etc/ipsec.secrets b/systemvm/debian/etc/ipsec.secrets similarity index 100% rename from systemvm/patches/vpn/etc/ipsec.secrets rename to systemvm/debian/etc/ipsec.secrets diff --git a/systemvm/patches/debian/etc/iptables/iptables-consoleproxy b/systemvm/debian/etc/iptables/iptables-consoleproxy similarity index 100% rename from systemvm/patches/debian/etc/iptables/iptables-consoleproxy rename to systemvm/debian/etc/iptables/iptables-consoleproxy diff --git a/systemvm/patches/debian/etc/iptables/iptables-dhcpsrvr b/systemvm/debian/etc/iptables/iptables-dhcpsrvr similarity index 100% rename from systemvm/patches/debian/etc/iptables/iptables-dhcpsrvr rename to systemvm/debian/etc/iptables/iptables-dhcpsrvr diff --git a/systemvm/patches/debian/etc/iptables/iptables-elbvm b/systemvm/debian/etc/iptables/iptables-elbvm similarity index 100% rename from systemvm/patches/debian/etc/iptables/iptables-elbvm rename to systemvm/debian/etc/iptables/iptables-elbvm diff --git a/systemvm/patches/debian/etc/iptables/iptables-ilbvm b/systemvm/debian/etc/iptables/iptables-ilbvm similarity index 100% rename from systemvm/patches/debian/etc/iptables/iptables-ilbvm rename to systemvm/debian/etc/iptables/iptables-ilbvm diff --git a/systemvm/patches/debian/etc/iptables/iptables-router b/systemvm/debian/etc/iptables/iptables-router similarity index 100% rename from systemvm/patches/debian/etc/iptables/iptables-router rename to systemvm/debian/etc/iptables/iptables-router diff --git a/systemvm/patches/debian/etc/iptables/iptables-secstorage b/systemvm/debian/etc/iptables/iptables-secstorage similarity index 100% rename from systemvm/patches/debian/etc/iptables/iptables-secstorage rename to systemvm/debian/etc/iptables/iptables-secstorage diff --git a/systemvm/patches/debian/etc/iptables/iptables-vpcrouter b/systemvm/debian/etc/iptables/iptables-vpcrouter similarity index 100% rename from systemvm/patches/debian/etc/iptables/iptables-vpcrouter rename to systemvm/debian/etc/iptables/iptables-vpcrouter diff --git a/systemvm/patches/debian/etc/iptables/rt_tables_init b/systemvm/debian/etc/iptables/rt_tables_init similarity index 100% rename from systemvm/patches/debian/etc/iptables/rt_tables_init rename to systemvm/debian/etc/iptables/rt_tables_init diff --git a/systemvm/patches/debian/etc/iptables/rules b/systemvm/debian/etc/iptables/rules similarity index 100% rename from systemvm/patches/debian/etc/iptables/rules rename to systemvm/debian/etc/iptables/rules diff --git a/systemvm/debian/etc/issue b/systemvm/debian/etc/issue new file mode 100644 index 00000000000..2d2b7c0f2f1 --- /dev/null +++ b/systemvm/debian/etc/issue @@ -0,0 +1,3 @@ + __?.o/ Apache CloudStack SystemVM 4.11 + ( )# https://cloudstack.apache.org + (___(_) \s \r \n \l diff --git a/systemvm/patches/debian/etc/logrotate.conf b/systemvm/debian/etc/logrotate.conf similarity index 100% rename from systemvm/patches/debian/etc/logrotate.conf rename to systemvm/debian/etc/logrotate.conf diff --git a/systemvm/patches/debian/etc/logrotate.d/apache2 b/systemvm/debian/etc/logrotate.d/apache2 similarity index 100% rename from systemvm/patches/debian/etc/logrotate.d/apache2 rename to systemvm/debian/etc/logrotate.d/apache2 diff --git a/systemvm/patches/debian/etc/logrotate.d/cloud b/systemvm/debian/etc/logrotate.d/cloud similarity index 100% rename from systemvm/patches/debian/etc/logrotate.d/cloud rename to systemvm/debian/etc/logrotate.d/cloud diff --git a/systemvm/patches/debian/etc/logrotate.d/conntrackd b/systemvm/debian/etc/logrotate.d/conntrackd similarity index 100% rename from systemvm/patches/debian/etc/logrotate.d/conntrackd rename to systemvm/debian/etc/logrotate.d/conntrackd diff --git a/systemvm/patches/debian/etc/logrotate.d/dnsmasq b/systemvm/debian/etc/logrotate.d/dnsmasq similarity index 100% rename from systemvm/patches/debian/etc/logrotate.d/dnsmasq rename to systemvm/debian/etc/logrotate.d/dnsmasq diff --git a/systemvm/patches/debian/etc/logrotate.d/haproxy b/systemvm/debian/etc/logrotate.d/haproxy similarity index 100% rename from systemvm/patches/debian/etc/logrotate.d/haproxy rename to systemvm/debian/etc/logrotate.d/haproxy diff --git a/systemvm/patches/debian/etc/logrotate.d/ppp b/systemvm/debian/etc/logrotate.d/ppp similarity index 100% rename from systemvm/patches/debian/etc/logrotate.d/ppp rename to systemvm/debian/etc/logrotate.d/ppp diff --git a/systemvm/patches/debian/etc/logrotate.d/rsyslog b/systemvm/debian/etc/logrotate.d/rsyslog similarity index 100% rename from systemvm/patches/debian/etc/logrotate.d/rsyslog rename to systemvm/debian/etc/logrotate.d/rsyslog diff --git a/systemvm/patches/debian/etc/modprobe.d/aesni_intel.conf b/systemvm/debian/etc/modprobe.d/aesni_intel.conf similarity index 100% rename from systemvm/patches/debian/etc/modprobe.d/aesni_intel.conf rename to systemvm/debian/etc/modprobe.d/aesni_intel.conf diff --git a/systemvm/patches/debian/etc/modprobe.d/pcspkr.conf b/systemvm/debian/etc/modprobe.d/pcspkr.conf similarity index 100% rename from systemvm/patches/debian/etc/modprobe.d/pcspkr.conf rename to systemvm/debian/etc/modprobe.d/pcspkr.conf diff --git a/systemvm/patches/vpn/etc/ppp/options.xl2tpd b/systemvm/debian/etc/ppp/options.xl2tpd similarity index 100% rename from systemvm/patches/vpn/etc/ppp/options.xl2tpd rename to systemvm/debian/etc/ppp/options.xl2tpd diff --git a/systemvm/patches/debian/etc/profile.d/cloud.sh b/systemvm/debian/etc/profile.d/cloud.sh similarity index 100% rename from systemvm/patches/debian/etc/profile.d/cloud.sh rename to systemvm/debian/etc/profile.d/cloud.sh diff --git a/systemvm/debian/etc/rc.local b/systemvm/debian/etc/rc.local new file mode 100755 index 00000000000..94c6466387c --- /dev/null +++ b/systemvm/debian/etc/rc.local @@ -0,0 +1,47 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[ ! -f /var/cache/cloud/enabled_svcs ] && touch /var/cache/cloud/enabled_svcs +for svc in $(cat /var/cache/cloud/enabled_svcs) +do + logger -t cloud "Starting $svc" + systemctl enable --no-block --now $svc +done + +[ ! -f /var/cache/cloud/disabled_svcs ] && touch /var/cache/cloud/disabled_svcs +for svc in $(cat /var/cache/cloud/disabled_svcs) +do + logger -t cloud "Stopping $svc" + systemctl disable --no-block --now $svc +done + +# Restore the persistent iptables nat, rules and filters for IPv4 and IPv6 if they exist +ipv4="/etc/iptables/rules.v4" +if [ -e $ipv4 ] +then + iptables-restore < $ipv4 +fi + +ipv6="/etc/iptables/rules.v6" +if [ -e $ipv6 ] +then + iptables-restore < $ipv6 +fi + +date > /var/cache/cloud/boot_up_done +logger -t cloud "Boot up process done" diff --git a/systemvm/patches/debian/etc/rsyslog.conf b/systemvm/debian/etc/rsyslog.conf similarity index 100% rename from systemvm/patches/debian/etc/rsyslog.conf rename to systemvm/debian/etc/rsyslog.conf diff --git a/systemvm/patches/debian/etc/ssh/sshd_config b/systemvm/debian/etc/ssh/sshd_config similarity index 99% rename from systemvm/patches/debian/etc/ssh/sshd_config rename to systemvm/debian/etc/ssh/sshd_config index 4931cdb028a..f2775747305 100644 --- a/systemvm/patches/debian/etc/ssh/sshd_config +++ b/systemvm/debian/etc/ssh/sshd_config @@ -57,7 +57,6 @@ AuthorizedKeysFile .ssh/authorized_keys #IgnoreRhosts yes # To disable tunneled clear text passwords, change to no here! -#PasswordAuthentication yes #PermitEmptyPasswords no PasswordAuthentication no diff --git a/systemvm/patches/debian/etc/sysctl.conf b/systemvm/debian/etc/sysctl.conf similarity index 100% rename from systemvm/patches/debian/etc/sysctl.conf rename to systemvm/debian/etc/sysctl.conf diff --git a/systemvm/debian/etc/systemd/system/cloud-early-config.service b/systemvm/debian/etc/systemd/system/cloud-early-config.service new file mode 100644 index 00000000000..2af52767cc3 --- /dev/null +++ b/systemvm/debian/etc/systemd/system/cloud-early-config.service @@ -0,0 +1,18 @@ +[Unit] +Description=CloudStack post-boot patching service using cmdline +DefaultDependencies=no + +Before=network-pre.target +Wants=network-pre.target + +Requires=local-fs.target +After=local-fs.target + +[Install] +WantedBy=multi-user.target + +[Service] +Type=oneshot +ExecStart=/opt/cloud/bin/setup/cloud-early-config +RemainAfterExit=true +TimeoutStartSec=5min diff --git a/systemvm/debian/etc/systemd/system/cloud-password-server@.service b/systemvm/debian/etc/systemd/system/cloud-password-server@.service new file mode 100644 index 00000000000..2fe8bec11dd --- /dev/null +++ b/systemvm/debian/etc/systemd/system/cloud-password-server@.service @@ -0,0 +1,12 @@ +[Unit] +Description=Cloud password server on %I +After=network.target local-fs.target + +[Install] +WantedBy=multi-user.target + +[Service] +Type=simple +WorkingDirectory=/opt/cloud/bin +ExecStart=/opt/cloud/bin/passwd_server_ip.py %I +Restart=on-failure diff --git a/systemvm/debian/etc/systemd/system/cloud-postinit.service b/systemvm/debian/etc/systemd/system/cloud-postinit.service new file mode 100644 index 00000000000..83f0636a72d --- /dev/null +++ b/systemvm/debian/etc/systemd/system/cloud-postinit.service @@ -0,0 +1,14 @@ +[Unit] +Description=CloudStack post-patching init script +After=cloud-early-config.service network.target local-fs.target +Before=ssh.service +Requires=network.service + +[Install] +WantedBy=multi-user.target + +[Service] +Type=oneshot +ExecStart=/opt/cloud/bin/setup/postinit.sh +RemainAfterExit=true +TimeoutStartSec=1min diff --git a/systemvm/debian/etc/systemd/system/cloud.service b/systemvm/debian/etc/systemd/system/cloud.service new file mode 100644 index 00000000000..9c61be46eba --- /dev/null +++ b/systemvm/debian/etc/systemd/system/cloud.service @@ -0,0 +1,13 @@ +[Unit] +Description=CloudStack Agent service +After=cloud-early-config.service network.target local-fs.target + +[Install] +WantedBy=multi-user.target + +[Service] +Type=simple +WorkingDirectory=/usr/local/cloud/systemvm +ExecStart=/usr/local/cloud/systemvm/_run.sh +Restart=always +RestartSec=5 diff --git a/systemvm/patches/debian/etc/vpcdnsmasq.conf b/systemvm/debian/etc/vpcdnsmasq.conf similarity index 100% rename from systemvm/patches/debian/etc/vpcdnsmasq.conf rename to systemvm/debian/etc/vpcdnsmasq.conf diff --git a/systemvm/patches/vpn/etc/xl2tpd/xl2tpd.conf b/systemvm/debian/etc/xl2tpd/xl2tpd.conf similarity index 100% rename from systemvm/patches/vpn/etc/xl2tpd/xl2tpd.conf rename to systemvm/debian/etc/xl2tpd/xl2tpd.conf diff --git a/systemvm/patches/debian/opt/cloud/bin/baremetal-vr.py b/systemvm/debian/opt/cloud/bin/baremetal-vr.py similarity index 99% rename from systemvm/patches/debian/opt/cloud/bin/baremetal-vr.py rename to systemvm/debian/opt/cloud/bin/baremetal-vr.py index e5b6aca51d6..20352ddeeab 100755 --- a/systemvm/patches/debian/opt/cloud/bin/baremetal-vr.py +++ b/systemvm/debian/opt/cloud/bin/baremetal-vr.py @@ -15,8 +15,6 @@ #specific language governing permissions and limitations #under the License. -__author__ = 'frank' - import subprocess import urllib import hmac diff --git a/systemvm/patches/debian/opt/cloud/bin/baremetal_snat.sh b/systemvm/debian/opt/cloud/bin/baremetal_snat.sh similarity index 100% rename from systemvm/patches/debian/opt/cloud/bin/baremetal_snat.sh rename to systemvm/debian/opt/cloud/bin/baremetal_snat.sh diff --git a/systemvm/patches/debian/opt/cloud/bin/bumpup_priority.sh b/systemvm/debian/opt/cloud/bin/bumpup_priority.sh similarity index 100% rename from systemvm/patches/debian/opt/cloud/bin/bumpup_priority.sh rename to systemvm/debian/opt/cloud/bin/bumpup_priority.sh diff --git a/systemvm/patches/debian/opt/cloud/bin/checkbatchs2svpn.sh b/systemvm/debian/opt/cloud/bin/checkbatchs2svpn.sh similarity index 100% rename from systemvm/patches/debian/opt/cloud/bin/checkbatchs2svpn.sh rename to systemvm/debian/opt/cloud/bin/checkbatchs2svpn.sh diff --git a/systemvm/patches/debian/opt/cloud/bin/checkrouter.sh b/systemvm/debian/opt/cloud/bin/checkrouter.sh similarity index 100% rename from systemvm/patches/debian/opt/cloud/bin/checkrouter.sh rename to systemvm/debian/opt/cloud/bin/checkrouter.sh diff --git a/systemvm/patches/debian/opt/cloud/bin/checks2svpn.sh b/systemvm/debian/opt/cloud/bin/checks2svpn.sh similarity index 100% rename from systemvm/patches/debian/opt/cloud/bin/checks2svpn.sh rename to systemvm/debian/opt/cloud/bin/checks2svpn.sh diff --git a/systemvm/patches/debian/opt/cloud/bin/cloud-nic.sh b/systemvm/debian/opt/cloud/bin/cloud-nic.sh similarity index 100% rename from systemvm/patches/debian/opt/cloud/bin/cloud-nic.sh rename to systemvm/debian/opt/cloud/bin/cloud-nic.sh diff --git a/systemvm/patches/debian/opt/cloud/bin/configure.py b/systemvm/debian/opt/cloud/bin/configure.py similarity index 85% rename from systemvm/patches/debian/opt/cloud/bin/configure.py rename to systemvm/debian/opt/cloud/bin/configure.py index e84e0171d30..a3b7674926c 100755 --- a/systemvm/patches/debian/opt/cloud/bin/configure.py +++ b/systemvm/debian/opt/cloud/bin/configure.py @@ -16,16 +16,14 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -import sys import base64 +import logging +import os +import re +import sys +import time from collections import OrderedDict - -import logging -import re - -import os.path -import os from fcntl import flock, LOCK_EX, LOCK_UN from cs.CsDatabag import CsDataBag @@ -39,11 +37,10 @@ from cs.CsConfig import CsConfig from cs.CsProcess import CsProcess from cs.CsStaticRoutes import CsStaticRoutes -OCCURRENCES = 1 class CsPassword(CsDataBag): - TOKEN_FILE="/tmp/passwdsrvrtoken" + TOKEN_FILE = "/tmp/passwdsrvrtoken" def process(self): for item in self.dbag: @@ -66,7 +63,7 @@ class CsPassword(CsDataBag): proc = CsProcess(['/opt/cloud/bin/passwd_server_ip.py', server_ip]) if proc.find(): update_command = 'curl --header "DomU_Request: save_password" "http://{SERVER_IP}:8080/" -F "ip={VM_IP}" -F "password={PASSWORD}" ' \ - '-F "token={TOKEN}" >/dev/null 2>/dev/null &'.format(SERVER_IP=server_ip, VM_IP=vm_ip, PASSWORD=password, TOKEN=token) + '-F "token={TOKEN}" >/dev/null 2>/dev/null &'.format(SERVER_IP=server_ip, VM_IP=vm_ip, PASSWORD=password, TOKEN=token) result = CsHelper.execute(update_command) logging.debug("Update password server result ==> %s" % result) @@ -96,7 +93,7 @@ class CsAcl(CsDataBag): self.rule['allowed'] = True self.rule['action'] = "ACCEPT" - if self.rule['type'] == 'all' and obj['source_cidr_list']: + if self.rule['type'] == 'all' and obj['source_cidr_list']: self.rule['cidr'] = [] else: self.rule['cidr'] = obj['source_cidr_list'] @@ -127,7 +124,7 @@ class CsAcl(CsDataBag): rnge = '' if "first_port" in self.rule.keys() and \ self.rule['first_port'] == self.rule['last_port']: - rnge = " --dport %s " %self.rule['first_port'] + rnge = " --dport %s " % self.rule['first_port'] if "first_port" in self.rule.keys() and \ self.rule['first_port'] != self.rule['last_port']: rnge = " --dport %s:%s" % (rule['first_port'], rule['last_port']) @@ -149,21 +146,21 @@ class CsAcl(CsDataBag): " -p %s " % rule['protocol'] + " %s -j %s" % (rnge, self.rule['action'])]) - sflag=False - dflag=False + sflag = False + dflag = False if self.direction == 'egress': ruleId = self.rule['id'] - sourceIpsetName = 'sourceCidrIpset-%d' %ruleId - destIpsetName = 'destCidrIpset-%d' %ruleId + sourceIpsetName = 'sourceCidrIpset-%d' % ruleId + destIpsetName = 'destCidrIpset-%d' % ruleId - #create source cidr ipset + # Create source cidr ipset srcIpset = 'ipset create '+sourceIpsetName + ' hash:net ' dstIpset = 'ipset create '+destIpsetName + ' hash:net ' CsHelper.execute(srcIpset) CsHelper.execute(dstIpset) for cidr in self.rule['cidr']: - ipsetAddCmd = 'ipset add '+ sourceIpsetName + ' '+cidr + ipsetAddCmd = 'ipset add ' + sourceIpsetName + ' ' + cidr CsHelper.execute(ipsetAddCmd) sflag = True @@ -171,7 +168,7 @@ class CsAcl(CsDataBag): for cidr in self.rule['dcidr']: if cidr == CIDR_ALL: continue - ipsetAddCmd = 'ipset add '+ destIpsetName + ' '+cidr + ipsetAddCmd = 'ipset add ' + destIpsetName + ' ' + cidr CsHelper.execute(ipsetAddCmd) dflag = True @@ -196,13 +193,13 @@ class CsAcl(CsDataBag): else: self.rule['action'] = "ACCEPT" - egressIpsetStr='' - if sflag == True and dflag == True: + egressIpsetStr = '' + if sflag and dflag: egressIpsetStr = " -m set --match-set %s src " % sourceIpsetName + \ " -m set --match-set %s dst " % destIpsetName - elif sflag == True: + elif sflag: egressIpsetStr = " -m set --match-set %s src " % sourceIpsetName - elif dflag == True: + elif dflag: egressIpsetStr = " -m set --match-set %s dst " % destIpsetName if rule['protocol'] == "icmp": @@ -359,7 +356,7 @@ class CsVmMetadata(CsDataBag): if data is not None: # need to pad data if it is not valid base 64 if len(data) % 4 != 0: - data += (4-(len(data) % 4)) * "=" + data += (4 - (len(data) % 4)) * "=" data = base64.b64decode(data) fh = open(dest, "w") @@ -466,7 +463,7 @@ class CsVmMetadata(CsDataBag): def __unflock(self, file): try: flock(file, LOCK_UN) - except IOError: + except IOError as e: print "failed to unlock file" + file.name + " due to : " + e.strerror sys.exit(1) # FIXME return True @@ -539,8 +536,8 @@ class CsSite2SiteVpn(CsDataBag): peerlist = obj['peer_guest_cidr_list'].replace(' ', '') vpnconffile = "%s/ipsec.vpn-%s.conf" % (self.VPNCONFDIR, rightpeer) vpnsecretsfile = "%s/ipsec.vpn-%s.secrets" % (self.VPNCONFDIR, rightpeer) - ikepolicy=obj['ike_policy'].replace(';','-') - esppolicy=obj['esp_policy'].replace(';','-') + ikepolicy = obj['ike_policy'].replace(';', '-') + esppolicy = obj['esp_policy'].replace(';', '-') if rightpeer in self.confips: self.confips.remove(rightpeer) @@ -561,7 +558,7 @@ class CsSite2SiteVpn(CsDataBag): file.addeq(" keyingtries=2") file.addeq(" auto=route") if 'encap' not in obj: - obj['encap']=False + obj['encap'] = False file.addeq(" forceencaps=%s" % CsHelper.bool_to_yn(obj['encap'])) if obj['dpd']: file.addeq(" dpddelay=30") @@ -595,14 +592,14 @@ class CsSite2SiteVpn(CsDataBag): class CsVpnUser(CsDataBag): - PPP_CHAP='/etc/ppp/chap-secrets' + PPP_CHAP = '/etc/ppp/chap-secrets' def process(self): for user in self.dbag: if user == 'id': continue - userconfig=self.dbag[user] + userconfig = self.dbag[user] if userconfig['add']: self.add_l2tp_ipsec_user(user, userconfig) else: @@ -612,7 +609,7 @@ class CsVpnUser(CsDataBag): userfound = False password = obj['password'] - userAddEntry = "%s * %s *" %(user,password) + userAddEntry = "%s * %s *" % (user, password) logging.debug("Adding vpn user '%s'" % user) file = CsFile(self.PPP_CHAP) @@ -626,7 +623,7 @@ class CsVpnUser(CsDataBag): def del_l2tp_ipsec_user(self, user, obj): userfound = False password = obj['password'] - userentry = "%s * %s *" % (user,password) + userentry = "%s * %s *" % (user, password) logging.debug("Deleting the user '%s'" % user) file = CsFile(self.PPP_CHAP) @@ -647,7 +644,7 @@ class CsVpnUser(CsDataBag): if pppd == 'PPPD_PID': pid = str.split('=')[1] if pid: - logging.debug("killing process %s" %pid) + logging.debug("killing process %s" % pid) CsHelper.execute('kill -9 %s' % pid) @@ -662,12 +659,11 @@ class CsRemoteAccessVpn(CsDataBag): for public_ip in self.dbag: if public_ip == "id": continue - vpnconfig=self.dbag[public_ip] + vpnconfig = self.dbag[public_ip] # Enable remote access vpn if vpnconfig['create']: - shutdownIpsec = False - logging.debug("Enabling remote access vpn on "+ public_ip) + logging.debug("Enabling remote access vpn on " + public_ip) CsHelper.start_if_stopped("ipsec") self.configure_l2tpIpsec(public_ip, self.dbag[public_ip]) @@ -682,20 +678,19 @@ class CsRemoteAccessVpn(CsDataBag): CsHelper.execute("ipsec down L2TP-PSK") CsHelper.execute("systemctl stop xl2tpd") + def configure_l2tpIpsec(self, left, obj): + l2tpconffile = "%s/l2tp.conf" % (self.VPNCONFDIR) + vpnsecretfilte = "%s/ipsec.any.secrets" % (self.VPNCONFDIR) + xl2tpdconffile = "/etc/xl2tpd/xl2tpd.conf" + xl2tpoptionsfile = "/etc/ppp/options.xl2tpd" - def configure_l2tpIpsec(self, left, obj): - l2tpconffile="%s/l2tp.conf" % (self.VPNCONFDIR) - vpnsecretfilte="%s/ipsec.any.secrets" % (self.VPNCONFDIR) - xl2tpdconffile="/etc/xl2tpd/xl2tpd.conf" - xl2tpoptionsfile='/etc/ppp/options.xl2tpd' + localip = obj['local_ip'] + localcidr = obj['local_cidr'] + publicIface = obj['public_interface'] + iprange = obj['ip_range'] + psk = obj['preshared_key'] - localip=obj['local_ip'] - localcidr=obj['local_cidr'] - publicIface=obj['public_interface'] - iprange=obj['ip_range'] - psk=obj['preshared_key'] - - #left + # Left l2tpfile = CsFile(l2tpconffile) l2tpfile.addeq(" left=%s" % left) l2tpfile.commit() @@ -706,19 +701,18 @@ class CsRemoteAccessVpn(CsDataBag): secret.commit() xl2tpdconf = CsFile(xl2tpdconffile) - xl2tpdconf.addeq("ip range = %s" %iprange) - xl2tpdconf.addeq("local ip = %s" %localip) + xl2tpdconf.addeq("ip range = %s" % iprange) + xl2tpdconf.addeq("local ip = %s" % localip) xl2tpdconf.commit() - xl2tpoptions=CsFile(xl2tpoptionsfile) - xl2tpoptions.search("ms-dns ", "ms-dns %s" %localip) + xl2tpoptions = CsFile(xl2tpoptionsfile) + xl2tpoptions.search("ms-dns ", "ms-dns %s" % localip) xl2tpoptions.commit() def remoteaccessvpn_iptables(self, publicip, obj): - publicdev=obj['public_interface'] - localcidr=obj['local_cidr'] - local_ip=obj['local_ip'] - + publicdev = obj['public_interface'] + localcidr = obj['local_cidr'] + local_ip = obj['local_ip'] self.fw.append(["", "", "-A INPUT -i %s --dst %s -p udp -m udp --dport 500 -j ACCEPT" % (publicdev, publicip)]) self.fw.append(["", "", "-A INPUT -i %s --dst %s -p udp -m udp --dport 4500 -j ACCEPT" % (publicdev, publicip)]) @@ -729,31 +723,30 @@ class CsRemoteAccessVpn(CsDataBag): self.fw.append(["", "", "-A OUTPUT -p esp -j ACCEPT"]) if self.config.is_vpc(): - self.fw.append(["", ""," -N VPN_FORWARD"]) - self.fw.append(["", "","-I FORWARD -i ppp+ -j VPN_FORWARD"]) - self.fw.append(["", "","-I FORWARD -o ppp+ -j VPN_FORWARD"]) - self.fw.append(["", "","-I FORWARD -o ppp+ -j VPN_FORWARD"]) - self.fw.append(["", "","-A VPN_FORWARD -s %s -j RETURN" %localcidr]) - self.fw.append(["", "","-A VPN_FORWARD -i ppp+ -d %s -j RETURN" %localcidr]) - self.fw.append(["", "","-A VPN_FORWARD -i ppp+ -o ppp+ -j RETURN"]) + self.fw.append(["", "", " -N VPN_FORWARD"]) + self.fw.append(["", "", "-I FORWARD -i ppp+ -j VPN_FORWARD"]) + self.fw.append(["", "", "-I FORWARD -o ppp+ -j VPN_FORWARD"]) + self.fw.append(["", "", "-I FORWARD -o ppp+ -j VPN_FORWARD"]) + self.fw.append(["", "", "-A VPN_FORWARD -s %s -j RETURN" % localcidr]) + self.fw.append(["", "", "-A VPN_FORWARD -i ppp+ -d %s -j RETURN" % localcidr]) + self.fw.append(["", "", "-A VPN_FORWARD -i ppp+ -o ppp+ -j RETURN"]) else: - self.fw.append(["", "","-A FORWARD -i ppp+ -o ppp+ -j ACCEPT"]) - self.fw.append(["", "","-A FORWARD -s %s -o ppp+ -j ACCEPT" % localcidr]) - self.fw.append(["", "","-A FORWARD -i ppp+ -d %s -j ACCEPT" % localcidr]) + self.fw.append(["", "", "-A FORWARD -i ppp+ -o ppp+ -j ACCEPT"]) + self.fw.append(["", "", "-A FORWARD -s %s -o ppp+ -j ACCEPT" % localcidr]) + self.fw.append(["", "", "-A FORWARD -i ppp+ -d %s -j ACCEPT" % localcidr]) - - self.fw.append(["", "","-A INPUT -i ppp+ -m udp -p udp --dport 53 -j ACCEPT"]) - self.fw.append(["", "","-A INPUT -i ppp+ -m tcp -p tcp --dport 53 -j ACCEPT"]) - self.fw.append(["nat", "","-I PREROUTING -i ppp+ -m tcp --dport 53 -j DNAT --to-destination %s" % local_ip]) + self.fw.append(["", "", "-A INPUT -i ppp+ -m udp -p udp --dport 53 -j ACCEPT"]) + self.fw.append(["", "", "-A INPUT -i ppp+ -m tcp -p tcp --dport 53 -j ACCEPT"]) + self.fw.append(["nat", "", "-I PREROUTING -i ppp+ -m tcp --dport 53 -j DNAT --to-destination %s" % local_ip]) if self.config.is_vpc(): return - self.fw.append(["mangle", "","-N VPN_%s " %publicip]) - self.fw.append(["mangle", "","-A VPN_%s -j RETURN " % publicip]) - self.fw.append(["mangle", "","-I VPN_%s -p ah -j ACCEPT " % publicip]) - self.fw.append(["mangle", "","-I VPN_%s -p esp -j ACCEPT " % publicip]) - self.fw.append(["mangle", "","-I PREROUTING -d %s -j VPN_%s " % (publicip, publicip)]) + self.fw.append(["mangle", "", "-N VPN_%s " % publicip]) + self.fw.append(["mangle", "", "-A VPN_%s -j RETURN " % publicip]) + self.fw.append(["mangle", "", "-I VPN_%s -p ah -j ACCEPT " % publicip]) + self.fw.append(["mangle", "", "-I VPN_%s -p esp -j ACCEPT " % publicip]) + self.fw.append(["mangle", "", "-I PREROUTING -d %s -j VPN_%s " % (publicip, publicip)]) class CsForwardingRules(CsDataBag): @@ -768,7 +761,7 @@ class CsForwardingRules(CsDataBag): elif rule["type"] == "staticnat": self.processStaticNatRule(rule) - #return the VR guest interface ip + # Return the VR guest interface ip def getGuestIp(self): interfaces = [] ipAddr = None @@ -814,7 +807,7 @@ class CsForwardingRules(CsDataBag): self.forward_vr(rule) def forward_vr(self, rule): - #prefetch iptables variables + # Prefetch iptables variables public_fwinterface = self.getDeviceByIp(rule['public_ip']) internal_fwinterface = self.getDeviceByIp(rule['internal_ip']) public_fwports = self.portsToString(rule['public_ports'], ':') @@ -930,10 +923,10 @@ class CsForwardingRules(CsDataBag): raise Exception("Ip address %s has no device in the ips databag" % rule["public_ip"]) self.fw.append(["mangle", "", - "-I PREROUTING -s %s/32 -m state --state NEW -j CONNMARK --save-mark --nfmask 0xffffffff --ctmask 0xffffffff" % \ + "-I PREROUTING -s %s/32 -m state --state NEW -j CONNMARK --save-mark --nfmask 0xffffffff --ctmask 0xffffffff" % rule["internal_ip"]]) self.fw.append(["mangle", "", - "-I PREROUTING -s %s/32 -m state --state NEW -j MARK --set-xmark 0x%s/0xffffffff" % \ + "-I PREROUTING -s %s/32 -m state --state NEW -j MARK --set-xmark 0x%s/0xffffffff" % (rule["internal_ip"], device[len("eth"):])]) self.fw.append(["nat", "front", "-A PREROUTING -d %s/32 -j DNAT --to-destination %s" % (rule["public_ip"], rule["internal_ip"])]) @@ -944,11 +937,12 @@ class CsForwardingRules(CsDataBag): self.fw.append(["filter", "", "-A FORWARD -i %s -o eth0 -d %s -m state --state NEW -j ACCEPT " % (device, rule["internal_ip"])]) - #configure the hairpin nat + # Configure the hairpin nat self.fw.append(["nat", "front", "-A PREROUTING -d %s -i eth0 -j DNAT --to-destination %s" % (rule["public_ip"], rule["internal_ip"])]) - self.fw.append(["nat", "front", "-A POSTROUTING -s %s -d %s -j SNAT -o eth0 --to-source %s" % (self.getNetworkByIp(rule['internal_ip']),rule["internal_ip"], self.getGuestIp())]) + self.fw.append(["nat", "front", "-A POSTROUTING -s %s -d %s -j SNAT -o eth0 --to-source %s" % + (self.getNetworkByIp(rule['internal_ip']), rule["internal_ip"], self.getGuestIp())]) class IpTablesExecutor: @@ -984,8 +978,9 @@ class IpTablesExecutor: logging.debug("Configuring iptables rules done ...saving rules") # Save iptables configuration - will be loaded on reboot by the iptables-restore that is configured on /etc/rc.local - CsHelper.save_iptables("iptables-save", "/etc/iptables/router_rules.v4") - CsHelper.save_iptables("ip6tables-save", "/etc/iptables/router_rules.v6") + CsHelper.save_iptables("iptables-save", "/etc/iptables/rules.v4") + CsHelper.save_iptables("ip6tables-save", "/etc/iptables/rules.v6") + def main(argv): # The file we are currently processing, if it is "cmd_line.json" everything will be processed. @@ -995,6 +990,8 @@ def main(argv): logging.debug("No file was received, do not go on processing the other actions. Just leave for now.") return + json_type = os.path.basename(process_file).split('.json')[0] + # The "GLOBAL" Configuration object config = CsConfig() @@ -1009,53 +1006,51 @@ def main(argv): config.address().compare() config.address().process() - databag_map = OrderedDict([("guest_network.json", {"process_iptables" : True, "executor" : IpTablesExecutor(config)}), - ("vm_password.json", {"process_iptables" : False, "executor" : CsPassword("vmpassword", config)}), - ("vm_metadata.json", {"process_iptables" : False, "executor" : CsVmMetadata('vmdata', config)}), - ("network_acl.json", {"process_iptables" : True, "executor" : IpTablesExecutor(config)}), - ("firewall_rules.json", {"process_iptables" : True, "executor" : IpTablesExecutor(config)}), - ("forwarding_rules.json", {"process_iptables" : True, "executor" : IpTablesExecutor(config)}), - ("staticnat_rules.json", {"process_iptables" : True, "executor" : IpTablesExecutor(config)}), - ("site_2_site_vpn.json", {"process_iptables" : True, "executor" : IpTablesExecutor(config)}), - ("remote_access_vpn.json", {"process_iptables" : True, "executor" : IpTablesExecutor(config)}), - ("vpn_user_list.json", {"process_iptables" : False, "executor" : CsVpnUser("vpnuserlist", config)}), - ("vm_dhcp_entry.json", {"process_iptables" : False, "executor" : CsDhcp("dhcpentry", config)}), - ("dhcp.json", {"process_iptables" : False, "executor" : CsDhcp("dhcpentry", config)}), - ("load_balancer.json", {"process_iptables" : True, "executor" : IpTablesExecutor(config)}), - ("monitor_service.json", {"process_iptables" : False, "executor" : CsMonitor("monitorservice", config)}), - ("static_routes.json", {"process_iptables" : False, "executor" : CsStaticRoutes("staticroutes", config)}) + databag_map = OrderedDict([("guest_network", {"process_iptables": True, "executor": []}), + ("vm_password", {"process_iptables": False, "executor": [CsPassword("vmpassword", config)]}), + ("vm_metadata", {"process_iptables": False, "executor": [CsVmMetadata('vmdata', config)]}), + ("network_acl", {"process_iptables": True, "executor": []}), + ("firewall_rules", {"process_iptables": True, "executor": []}), + ("forwarding_rules", {"process_iptables": True, "executor": []}), + ("staticnat_rules", {"process_iptables": True, "executor": []}), + ("site_2_site_vpn", {"process_iptables": True, "executor": []}), + ("remote_access_vpn", {"process_iptables": True, "executor": []}), + ("vpn_user_list", {"process_iptables": False, "executor": [CsVpnUser("vpnuserlist", config)]}), + ("vm_dhcp_entry", {"process_iptables": False, "executor": [CsDhcp("dhcpentry", config)]}), + ("dhcp", {"process_iptables": False, "executor": [CsDhcp("dhcpentry", config)]}), + ("load_balancer", {"process_iptables": True, "executor": []}), + ("monitor_service", {"process_iptables": False, "executor": [CsMonitor("monitorservice", config)]}), + ("static_routes", {"process_iptables": False, "executor": [CsStaticRoutes("staticroutes", config)]}) ]) - if process_file.count("cmd_line.json") == OCCURRENCES: - logging.debug("cmd_line.json changed. All other files will be processed as well.") - - while databag_map: - item = databag_map.popitem(last = False) - item_name = item[0] - item_dict = item[1] - if not item_dict["process_iptables"]: - executor = item_dict["executor"] - executor.process() + def execDatabag(key, db): + if key not in db.keys() or 'executor' not in db[key]: + logging.warn("Unable to find config or executor(s) for the databag type %s" % key) + return + for executor in db[key]['executor']: + logging.debug("Processing for databag type: %s" % key) + executor.process() + def execIptables(config): + logging.debug("Processing iptables rules") iptables_executor = IpTablesExecutor(config) iptables_executor.process() + + if json_type == "cmd_line": + logging.debug("cmd_line.json changed. All other files will be processed as well.") + for key in databag_map.keys(): + execDatabag(key, databag_map) + execIptables(config) + elif json_type in databag_map.keys(): + execDatabag(json_type, databag_map) + if databag_map[json_type]['process_iptables']: + execIptables(config) else: - while databag_map: - item = databag_map.popitem(last = False) - item_name = item[0] - item_dict = item[1] - if process_file.count(item_name) == OCCURRENCES: - executor = item_dict["executor"] - executor.process() - - if item_dict["process_iptables"]: - iptables_executor = IpTablesExecutor(config) - iptables_executor.process() - - break + logging.warn("Unable to find and process databag for file: %s, for json type=%s" % (process_file, json_type)) red = CsRedundant(config) red.set() + return 0 if __name__ == "__main__": main(sys.argv) diff --git a/systemvm/patches/debian/opt/cloud/bin/cs/CsAddress.py b/systemvm/debian/opt/cloud/bin/cs/CsAddress.py similarity index 99% rename from systemvm/patches/debian/opt/cloud/bin/cs/CsAddress.py rename to systemvm/debian/opt/cloud/bin/cs/CsAddress.py index d190a723f0c..6ad50459b57 100755 --- a/systemvm/patches/debian/opt/cloud/bin/cs/CsAddress.py +++ b/systemvm/debian/opt/cloud/bin/cs/CsAddress.py @@ -27,6 +27,7 @@ from CsRule import CsRule VRRP_TYPES = ['guest'] + class CsAddress(CsDataBag): def compare(self): @@ -359,7 +360,6 @@ class CsIP: self.fw.append(["filter", "", "-P INPUT DROP"]) self.fw.append(["filter", "", "-P FORWARD DROP"]) - def fw_router(self): if self.config.is_vpc(): return @@ -439,7 +439,7 @@ class CsIP: if self.get_type() in ["guest"]: self.fw.append(["mangle", "front", "-A PREROUTING " + - " -i %s -m state --state RELATED,ESTABLISHED " % self.dev + + " -i %s -m state --state RELATED,ESTABLISHED " % self.dev + "-j CONNMARK --restore-mark --nfmask 0xffffffff --ctmask 0xffffffff"]) guestNetworkCidr = self.address['network'] self.fw.append(["filter", "", "-A FORWARD -d %s -o %s -j ACL_INBOUND_%s" % diff --git a/systemvm/patches/debian/opt/cloud/bin/cs/CsApp.py b/systemvm/debian/opt/cloud/bin/cs/CsApp.py similarity index 87% rename from systemvm/patches/debian/opt/cloud/bin/cs/CsApp.py rename to systemvm/debian/opt/cloud/bin/cs/CsApp.py index 496a0e7876a..2085088db8e 100755 --- a/systemvm/patches/debian/opt/cloud/bin/cs/CsApp.py +++ b/systemvm/debian/opt/cloud/bin/cs/CsApp.py @@ -16,7 +16,6 @@ # specific language governing permissions and limitations # under the License. import os -import CsHelper from CsFile import CsFile from CsProcess import CsProcess import CsHelper @@ -74,20 +73,13 @@ class CsPasswdSvc(): self.ip = ip def start(self): - proc = CsProcess(["dummy"]) - if proc.grep("passwd_server_ip %s" % self.ip) == -1: - proc.start("/opt/cloud/bin/passwd_server_ip %s >> /var/log/cloud.log 2>&1" % self.ip, "&") + CsHelper.service("cloud-password-server@%s" % self.ip, "start") def stop(self): - proc = CsProcess(["Password Service"]) - pid = proc.grep("passwd_server_ip %s" % self.ip) - proc.kill(pid) - pid = proc.grep("8080,reuseaddr,fork,crnl,bind=%s" % self.ip) - proc.kill(pid) + CsHelper.service("cloud-password-server@%s" % self.ip, "stop") def restart(self): - self.stop() - self.start() + CsHelper.service("cloud-password-server@%s" % self.ip, "restart") class CsDnsmasq(CsApp): diff --git a/systemvm/patches/debian/opt/cloud/bin/cs/CsConfig.py b/systemvm/debian/opt/cloud/bin/cs/CsConfig.py similarity index 100% rename from systemvm/patches/debian/opt/cloud/bin/cs/CsConfig.py rename to systemvm/debian/opt/cloud/bin/cs/CsConfig.py diff --git a/systemvm/patches/debian/opt/cloud/bin/cs/CsDatabag.py b/systemvm/debian/opt/cloud/bin/cs/CsDatabag.py similarity index 97% rename from systemvm/patches/debian/opt/cloud/bin/cs/CsDatabag.py rename to systemvm/debian/opt/cloud/bin/cs/CsDatabag.py index 9ccb768d14c..adb9a1aae74 100755 --- a/systemvm/patches/debian/opt/cloud/bin/cs/CsDatabag.py +++ b/systemvm/debian/opt/cloud/bin/cs/CsDatabag.py @@ -136,8 +136,8 @@ class CsCmdLine(CsDataBag): This is slightly difficult to happen, but if it does, destroy the router with the password generated with the code below and restart the VPC with out the clean up option. ''' - if(self.get_type()=='router'): - passwd="%s-%s" % (self.get_eth2_ip(), self.get_router_id()) + if self.get_type() == 'router': + passwd = "%s-%s" % (self.get_eth2_ip(), self.get_router_id()) else: passwd = "%s-%s" % (self.get_vpccidr(), self.get_router_id()) md5 = hashlib.md5() diff --git a/systemvm/patches/debian/opt/cloud/bin/cs/CsDhcp.py b/systemvm/debian/opt/cloud/bin/cs/CsDhcp.py similarity index 98% rename from systemvm/patches/debian/opt/cloud/bin/cs/CsDhcp.py rename to systemvm/debian/opt/cloud/bin/cs/CsDhcp.py index fbd1e053ce6..f6135010744 100755 --- a/systemvm/patches/debian/opt/cloud/bin/cs/CsDhcp.py +++ b/systemvm/debian/opt/cloud/bin/cs/CsDhcp.py @@ -135,13 +135,13 @@ class CsDhcp(CsDataBag): # with a splay of 60 hours to prevent storms lease = randint(700, 760) - if entry['default_entry'] == True: + if entry['default_entry']: self.cloud.add("%s,%s,%s,%sh" % (entry['mac_address'], entry['ipv4_address'], entry['host_name'], lease)) else: - tag = entry['ipv4_address'].replace(".","_") + tag = entry['ipv4_address'].replace(".", "_") self.cloud.add("%s,set:%s,%s,%s,%sh" % (entry['mac_address'], tag, entry['ipv4_address'], diff --git a/systemvm/patches/debian/opt/cloud/bin/cs/CsFile.py b/systemvm/debian/opt/cloud/bin/cs/CsFile.py similarity index 99% rename from systemvm/patches/debian/opt/cloud/bin/cs/CsFile.py rename to systemvm/debian/opt/cloud/bin/cs/CsFile.py index 334e6314bbe..b0a25916da9 100755 --- a/systemvm/patches/debian/opt/cloud/bin/cs/CsFile.py +++ b/systemvm/debian/opt/cloud/bin/cs/CsFile.py @@ -67,7 +67,6 @@ class CsFile: self.config = list(self.new_config) logging.info("Updated file in-cache configuration") - def dump(self): for line in self.new_config: print line @@ -134,13 +133,12 @@ class CsFile: return True return False - def searchString(self, search, ignoreLinesStartWith): found = False logging.debug("Searching for %s string " % search) for index, line in enumerate(self.new_config): - print ' line = ' +line + print ' line = ' + line if line.lstrip().startswith(ignoreLinesStartWith): continue if search in line: @@ -149,7 +147,6 @@ class CsFile: return found - def deleteLine(self, search): found = False logging.debug("Searching for %s to remove the line " % search) @@ -162,7 +159,6 @@ class CsFile: self.new_config = list(temp_config) - def compare(self, o): result = (isinstance(o, self.__class__) and set(self.config) == set(o.config)) logging.debug("Comparison of CsFiles content is ==> %s" % result) diff --git a/systemvm/patches/debian/opt/cloud/bin/cs/CsGuestNetwork.py b/systemvm/debian/opt/cloud/bin/cs/CsGuestNetwork.py similarity index 96% rename from systemvm/patches/debian/opt/cloud/bin/cs/CsGuestNetwork.py rename to systemvm/debian/opt/cloud/bin/cs/CsGuestNetwork.py index d23a870af69..9a94dc66360 100755 --- a/systemvm/patches/debian/opt/cloud/bin/cs/CsGuestNetwork.py +++ b/systemvm/debian/opt/cloud/bin/cs/CsGuestNetwork.py @@ -40,7 +40,7 @@ class CsGuestNetwork: return self.config.get_dns() dns = [] - if not self.config.use_extdns() and 'router_guest_gateway' in self.data: + if 'router_guest_gateway' in self.data and not self.config.use_extdns(): dns.append(self.data['router_guest_gateway']) if 'dns' in self.data: diff --git a/systemvm/patches/debian/opt/cloud/bin/cs/CsHelper.py b/systemvm/debian/opt/cloud/bin/cs/CsHelper.py similarity index 93% rename from systemvm/patches/debian/opt/cloud/bin/cs/CsHelper.py rename to systemvm/debian/opt/cloud/bin/cs/CsHelper.py index 7ff32ee6ad4..ad8480d0ed0 100755 --- a/systemvm/patches/debian/opt/cloud/bin/cs/CsHelper.py +++ b/systemvm/debian/opt/cloud/bin/cs/CsHelper.py @@ -21,16 +21,18 @@ for use in the configuration process """ import subprocess import logging +import sys import os.path import re import shutil from netaddr import * from pprint import pprint -PUBLIC_INTERFACES = {"router" : "eth2", "vpcrouter" : "eth1"} +PUBLIC_INTERFACES = {"router": "eth2", "vpcrouter": "eth1"} + +STATE_COMMANDS = {"router": "ip addr | grep eth0 | grep inet | wc -l | xargs bash -c 'if [ $0 == 2 ]; then echo \"MASTER\"; else echo \"BACKUP\"; fi'", + "vpcrouter": "ip addr | grep eth1 | grep state | awk '{print $9;}' | xargs bash -c 'if [ $0 == \"UP\" ]; then echo \"MASTER\"; else echo \"BACKUP\"; fi'"} -STATE_COMMANDS = {"router" : "ip addr | grep eth0 | grep inet | wc -l | xargs bash -c 'if [ $0 == 2 ]; then echo \"MASTER\"; else echo \"BACKUP\"; fi'", - "vpcrouter" : "ip addr | grep eth1 | grep state | awk '{print $9;}' | xargs bash -c 'if [ $0 == \"UP\" ]; then echo \"MASTER\"; else echo \"BACKUP\"; fi'"} def reconfigure_interfaces(router_config, interfaces): for interface in interfaces: @@ -52,6 +54,7 @@ def reconfigure_interfaces(router_config, interfaces): else: execute(cmd) + def is_mounted(name): for i in execute("mount"): vals = i.lstrip().split() @@ -242,6 +245,7 @@ def copy_if_needed(src, dest): return copy(src, dest) + def copy(src, dest): """ copy source to destination. @@ -249,6 +253,6 @@ def copy(src, dest): try: shutil.copy2(src, dest) except IOError: - logging.Error("Could not copy %s to %s" % (src, dest)) + logging.error("Could not copy %s to %s" % (src, dest)) else: logging.info("Copied %s to %s" % (src, dest)) diff --git a/systemvm/patches/debian/opt/cloud/bin/cs/CsLoadBalancer.py b/systemvm/debian/opt/cloud/bin/cs/CsLoadBalancer.py similarity index 100% rename from systemvm/patches/debian/opt/cloud/bin/cs/CsLoadBalancer.py rename to systemvm/debian/opt/cloud/bin/cs/CsLoadBalancer.py diff --git a/systemvm/patches/debian/opt/cloud/bin/cs/CsMonitor.py b/systemvm/debian/opt/cloud/bin/cs/CsMonitor.py similarity index 100% rename from systemvm/patches/debian/opt/cloud/bin/cs/CsMonitor.py rename to systemvm/debian/opt/cloud/bin/cs/CsMonitor.py diff --git a/systemvm/patches/debian/opt/cloud/bin/cs/CsNetfilter.py b/systemvm/debian/opt/cloud/bin/cs/CsNetfilter.py similarity index 96% rename from systemvm/patches/debian/opt/cloud/bin/cs/CsNetfilter.py rename to systemvm/debian/opt/cloud/bin/cs/CsNetfilter.py index 06c59ff65bb..a9ac0ad5262 100755 --- a/systemvm/patches/debian/opt/cloud/bin/cs/CsNetfilter.py +++ b/systemvm/debian/opt/cloud/bin/cs/CsNetfilter.py @@ -143,7 +143,7 @@ class CsNetfilters(object): # PASS 2: Create rules for fw in list: tupledFw = tuple(fw) - if tupledFw in ruleSet : + if tupledFw in ruleSet: logging.debug("Already processed : %s", tupledFw) continue @@ -173,12 +173,12 @@ class CsNetfilters(object): else: cpy = cpy.replace("-A %s" % new_rule.get_chain(), '-I %s %s' % (new_rule.get_chain(), fw[1])) ret = CsHelper.execute2("iptables -t %s %s" % (new_rule.get_table(), cpy)) - #There are some issues in this framework causing failures .. like adding a chain without checking it is present causing + # There are some issues in this framework causing failures .. like adding a chain without checking it is present causing # the failures. Also some of the rule like removeFromLoadBalancerRule is deleting rule and deleteLoadBalancerRule - #trying to delete which causes the failure. - #For now raising the log. - #TODO: Need to fix in the framework. - if ret.returncode != 0 : + # trying to delete which causes the failure. + # For now raising the log. + # TODO: Need to fix in the framework. + if ret.returncode != 0: error = ret.communicate()[0] logging.debug("iptables command got failed ... continuing") ruleSet.add(tupledFw) diff --git a/systemvm/patches/debian/opt/cloud/bin/cs/CsProcess.py b/systemvm/debian/opt/cloud/bin/cs/CsProcess.py similarity index 100% rename from systemvm/patches/debian/opt/cloud/bin/cs/CsProcess.py rename to systemvm/debian/opt/cloud/bin/cs/CsProcess.py diff --git a/systemvm/patches/debian/opt/cloud/bin/cs/CsRedundant.py b/systemvm/debian/opt/cloud/bin/cs/CsRedundant.py similarity index 96% rename from systemvm/patches/debian/opt/cloud/bin/cs/CsRedundant.py rename to systemvm/debian/opt/cloud/bin/cs/CsRedundant.py index 99c7458d32e..108f3377fdd 100755 --- a/systemvm/patches/debian/opt/cloud/bin/cs/CsRedundant.py +++ b/systemvm/debian/opt/cloud/bin/cs/CsRedundant.py @@ -42,6 +42,7 @@ from CsStaticRoutes import CsStaticRoutes import socket from time import sleep + class CsRedundant(object): CS_RAMDISK_DIR = "/ramdisk" @@ -74,10 +75,6 @@ class CsRedundant(object): def _redundant_off(self): CsHelper.service("conntrackd", "stop") CsHelper.service("keepalived", "stop") - #CsHelper.umount_tmpfs(self.CS_RAMDISK_DIR) - #CsHelper.rmdir(self.CS_RAMDISK_DIR) - CsHelper.rm(self.CONNTRACKD_CONF) - CsHelper.rm(self.KEEPALIVED_CONF) def _redundant_on(self): guest = self.address.get_guest_if() @@ -110,9 +107,10 @@ class CsRedundant(object): CsHelper.service("keepalived", "stop") return - #CsHelper.mkdir(self.CS_RAMDISK_DIR, 0755, False) - #CsHelper.mount_tmpfs(self.CS_RAMDISK_DIR) - #CsHelper.mkdir(self.CS_ROUTER_DIR, 0755, False) + # setup_router should execute this already: + # CsHelper.mkdir(self.CS_RAMDISK_DIR, 0755, False) + # CsHelper.mount_tmpfs(self.CS_RAMDISK_DIR) + # CsHelper.mkdir(self.CS_ROUTER_DIR, 0755, False) for s in self.CS_TEMPLATES: d = s if s.endswith(".templ"): @@ -143,7 +141,7 @@ class CsRedundant(object): keepalived_conf.greplace("[RROUTER_BIN_PATH]", self.CS_ROUTER_DIR) keepalived_conf.section("authentication {", "}", [ - " auth_type AH \n", " auth_pass %s\n" % self.cl.get_router_password()]) + " auth_type AH \n", " auth_pass %s\n" % self.cl.get_router_password()[:8]]) keepalived_conf.section( "virtual_ipaddress {", "}", self._collect_ips()) @@ -195,6 +193,7 @@ class CsRedundant(object): proc = CsProcess(['/usr/sbin/keepalived']) if not proc.find() or keepalived_conf.is_changed() or force_keepalived_restart: keepalived_conf.commit() + os.chmod(self.KEEPALIVED_CONF, 0o644) CsHelper.service("keepalived", "restart") def release_lock(self): @@ -371,10 +370,10 @@ class CsRedundant(object): lines = [] for interface in self.address.get_interfaces(): if interface.needs_vrrp(): - cmdline=self.config.get_cmdline_instance() + cmdline = self.config.get_cmdline_instance() if not interface.is_added(): continue - if(cmdline.get_type()=='router'): + if cmdline.get_type() == 'router': str = " %s brd %s dev %s\n" % (cmdline.get_guest_gw(), interface.get_broadcast(), interface.get_device()) else: str = " %s brd %s dev %s\n" % (interface.get_gateway_cidr(), interface.get_broadcast(), interface.get_device()) diff --git a/systemvm/patches/debian/opt/cloud/bin/cs/CsRoute.py b/systemvm/debian/opt/cloud/bin/cs/CsRoute.py similarity index 93% rename from systemvm/patches/debian/opt/cloud/bin/cs/CsRoute.py rename to systemvm/debian/opt/cloud/bin/cs/CsRoute.py index 56096c86466..02cc88117d8 100755 --- a/systemvm/patches/debian/opt/cloud/bin/cs/CsRoute.py +++ b/systemvm/debian/opt/cloud/bin/cs/CsRoute.py @@ -34,14 +34,13 @@ class CsRoute: tablename = self.get_tablename(devicename) str = "%s %s" % (tablenumber, tablename) filename = "/etc/iproute2/rt_tables" - logging.info( - "Adding route table: " + str + " to " + filename + " if not present ") + logging.info("Adding route table: " + str + " to " + filename + " if not present ") if not CsHelper.definedinfile(filename, str): - CsHelper.execute("sudo echo " + str + " >> /etc/iproute2/rt_tables") + CsHelper.execute("sudo echo " + str + " >> /etc/iproute2/rt_tables") # remove "from all table tablename" if exists, else it will interfer with # routing of unintended traffic if self.findRule("from all lookup " + tablename): - CsHelper.execute("sudo ip rule delete from all table " + tablename) + CsHelper.execute("sudo ip rule delete from all table " + tablename) def flush_table(self, tablename): CsHelper.execute("ip route flush table %s" % (tablename)) @@ -115,4 +114,4 @@ class CsRoute: for i in CsHelper.execute("ip rule show"): if rule in i.strip(): return True - return False \ No newline at end of file + return False diff --git a/systemvm/patches/debian/opt/cloud/bin/cs/CsRule.py b/systemvm/debian/opt/cloud/bin/cs/CsRule.py similarity index 98% rename from systemvm/patches/debian/opt/cloud/bin/cs/CsRule.py rename to systemvm/debian/opt/cloud/bin/cs/CsRule.py index 9c48768b35a..85953fe6561 100755 --- a/systemvm/patches/debian/opt/cloud/bin/cs/CsRule.py +++ b/systemvm/debian/opt/cloud/bin/cs/CsRule.py @@ -49,7 +49,7 @@ class CsRule: logging.info("Added fwmark rule for %s" % (self.table)) def delMark(self): - if self.findMark(): + if self.findMark(): cmd = "ip rule delete fwmark %s table %s" % (self.tableNo, self.table) CsHelper.execute(cmd) logging.info("Deleting fwmark rule for %s" % (self.table)) diff --git a/systemvm/patches/debian/opt/cloud/bin/cs/CsStaticRoutes.py b/systemvm/debian/opt/cloud/bin/cs/CsStaticRoutes.py similarity index 100% rename from systemvm/patches/debian/opt/cloud/bin/cs/CsStaticRoutes.py rename to systemvm/debian/opt/cloud/bin/cs/CsStaticRoutes.py diff --git a/systemvm/patches/debian/opt/cloud/bin/cs/__init__.py b/systemvm/debian/opt/cloud/bin/cs/__init__.py similarity index 100% rename from systemvm/patches/debian/opt/cloud/bin/cs/__init__.py rename to systemvm/debian/opt/cloud/bin/cs/__init__.py diff --git a/systemvm/patches/debian/opt/cloud/bin/cs_cmdline.py b/systemvm/debian/opt/cloud/bin/cs_cmdline.py similarity index 100% rename from systemvm/patches/debian/opt/cloud/bin/cs_cmdline.py rename to systemvm/debian/opt/cloud/bin/cs_cmdline.py diff --git a/systemvm/patches/debian/opt/cloud/bin/cs_dhcp.py b/systemvm/debian/opt/cloud/bin/cs_dhcp.py similarity index 100% rename from systemvm/patches/debian/opt/cloud/bin/cs_dhcp.py rename to systemvm/debian/opt/cloud/bin/cs_dhcp.py diff --git a/systemvm/patches/debian/opt/cloud/bin/cs_firewallrules.py b/systemvm/debian/opt/cloud/bin/cs_firewallrules.py similarity index 100% rename from systemvm/patches/debian/opt/cloud/bin/cs_firewallrules.py rename to systemvm/debian/opt/cloud/bin/cs_firewallrules.py diff --git a/systemvm/patches/debian/opt/cloud/bin/cs_forwardingrules.py b/systemvm/debian/opt/cloud/bin/cs_forwardingrules.py similarity index 100% rename from systemvm/patches/debian/opt/cloud/bin/cs_forwardingrules.py rename to systemvm/debian/opt/cloud/bin/cs_forwardingrules.py diff --git a/systemvm/patches/debian/opt/cloud/bin/cs_guestnetwork.py b/systemvm/debian/opt/cloud/bin/cs_guestnetwork.py similarity index 100% rename from systemvm/patches/debian/opt/cloud/bin/cs_guestnetwork.py rename to systemvm/debian/opt/cloud/bin/cs_guestnetwork.py diff --git a/systemvm/patches/debian/opt/cloud/bin/cs_ip.py b/systemvm/debian/opt/cloud/bin/cs_ip.py similarity index 92% rename from systemvm/patches/debian/opt/cloud/bin/cs_ip.py rename to systemvm/debian/opt/cloud/bin/cs_ip.py index f83bf298c4d..1e7b326a1ec 100755 --- a/systemvm/patches/debian/opt/cloud/bin/cs_ip.py +++ b/systemvm/debian/opt/cloud/bin/cs_ip.py @@ -18,6 +18,7 @@ from netaddr import * + def merge(dbag, ip): nic_dev_id = None for dev in dbag: @@ -31,7 +32,7 @@ def merge(dbag, ip): ipo = IPNetwork(ip['public_ip'] + '/' + ip['netmask']) if 'nic_dev_id' in ip: - nic_dev_id = ip['nic_dev_id'] + nic_dev_id = ip['nic_dev_id'] ip['device'] = 'eth' + str(nic_dev_id) ip['broadcast'] = str(ipo.broadcast) ip['cidr'] = str(ipo.ip) + '/' + str(ipo.prefixlen) @@ -45,7 +46,7 @@ def merge(dbag, ip): dbag[ip['device']] = [ip] else: if 'source_nat' in ip and ip['source_nat'] and ip['device'] in dbag and len(dbag[ip['device']]) > 0: - dbag[ip['device']].insert(0, ip) # make sure the source_nat ip is first (primary) on the device + dbag[ip['device']].insert(0, ip) # Make sure the source_nat ip is first (primary) on the device else: dbag.setdefault(ip['device'], []).append(ip) diff --git a/systemvm/patches/debian/opt/cloud/bin/cs_loadbalancer.py b/systemvm/debian/opt/cloud/bin/cs_loadbalancer.py similarity index 100% rename from systemvm/patches/debian/opt/cloud/bin/cs_loadbalancer.py rename to systemvm/debian/opt/cloud/bin/cs_loadbalancer.py diff --git a/systemvm/patches/debian/opt/cloud/bin/cs_monitorservice.py b/systemvm/debian/opt/cloud/bin/cs_monitorservice.py similarity index 100% rename from systemvm/patches/debian/opt/cloud/bin/cs_monitorservice.py rename to systemvm/debian/opt/cloud/bin/cs_monitorservice.py diff --git a/systemvm/patches/debian/opt/cloud/bin/cs_network_acl.py b/systemvm/debian/opt/cloud/bin/cs_network_acl.py similarity index 100% rename from systemvm/patches/debian/opt/cloud/bin/cs_network_acl.py rename to systemvm/debian/opt/cloud/bin/cs_network_acl.py diff --git a/systemvm/patches/debian/opt/cloud/bin/cs_remoteaccessvpn.py b/systemvm/debian/opt/cloud/bin/cs_remoteaccessvpn.py similarity index 100% rename from systemvm/patches/debian/opt/cloud/bin/cs_remoteaccessvpn.py rename to systemvm/debian/opt/cloud/bin/cs_remoteaccessvpn.py diff --git a/systemvm/patches/debian/opt/cloud/bin/cs_site2sitevpn.py b/systemvm/debian/opt/cloud/bin/cs_site2sitevpn.py similarity index 100% rename from systemvm/patches/debian/opt/cloud/bin/cs_site2sitevpn.py rename to systemvm/debian/opt/cloud/bin/cs_site2sitevpn.py diff --git a/systemvm/patches/debian/opt/cloud/bin/cs_staticroutes.py b/systemvm/debian/opt/cloud/bin/cs_staticroutes.py similarity index 100% rename from systemvm/patches/debian/opt/cloud/bin/cs_staticroutes.py rename to systemvm/debian/opt/cloud/bin/cs_staticroutes.py diff --git a/systemvm/patches/debian/opt/cloud/bin/cs_vmdata.py b/systemvm/debian/opt/cloud/bin/cs_vmdata.py similarity index 100% rename from systemvm/patches/debian/opt/cloud/bin/cs_vmdata.py rename to systemvm/debian/opt/cloud/bin/cs_vmdata.py diff --git a/systemvm/patches/debian/opt/cloud/bin/cs_vmp.py b/systemvm/debian/opt/cloud/bin/cs_vmp.py similarity index 99% rename from systemvm/patches/debian/opt/cloud/bin/cs_vmp.py rename to systemvm/debian/opt/cloud/bin/cs_vmp.py index 7873fe897f5..d0169f733cf 100755 --- a/systemvm/patches/debian/opt/cloud/bin/cs_vmp.py +++ b/systemvm/debian/opt/cloud/bin/cs_vmp.py @@ -18,6 +18,7 @@ from pprint import pprint from netaddr import * + def merge(dbag, data): """ create a dictionary of values new in data ignoring those in dbag and return it diff --git a/systemvm/patches/debian/opt/cloud/bin/cs_vpnusers.py b/systemvm/debian/opt/cloud/bin/cs_vpnusers.py similarity index 96% rename from systemvm/patches/debian/opt/cloud/bin/cs_vpnusers.py rename to systemvm/debian/opt/cloud/bin/cs_vpnusers.py index 316fabc07d3..8dc454822f8 100755 --- a/systemvm/patches/debian/opt/cloud/bin/cs_vpnusers.py +++ b/systemvm/debian/opt/cloud/bin/cs_vpnusers.py @@ -38,8 +38,8 @@ def merge(dbag, data): del(dbagc[user]) for user in data['vpn_users']: - username=user['user'] - add=user['add'] + username = user['user'] + add = user['add'] if username not in dbagc.keys(): dbagc[username] = user elif username in dbagc.keys() and not add: diff --git a/systemvm/patches/debian/opt/cloud/bin/dnsmasq.sh b/systemvm/debian/opt/cloud/bin/dnsmasq.sh similarity index 100% rename from systemvm/patches/debian/opt/cloud/bin/dnsmasq.sh rename to systemvm/debian/opt/cloud/bin/dnsmasq.sh diff --git a/systemvm/patches/debian/opt/cloud/bin/edithosts.sh b/systemvm/debian/opt/cloud/bin/edithosts.sh similarity index 100% rename from systemvm/patches/debian/opt/cloud/bin/edithosts.sh rename to systemvm/debian/opt/cloud/bin/edithosts.sh diff --git a/systemvm/patches/debian/opt/cloud/bin/getRouterAlerts.sh b/systemvm/debian/opt/cloud/bin/getRouterAlerts.sh similarity index 100% rename from systemvm/patches/debian/opt/cloud/bin/getRouterAlerts.sh rename to systemvm/debian/opt/cloud/bin/getRouterAlerts.sh diff --git a/systemvm/patches/debian/opt/cloud/bin/get_template_version.sh b/systemvm/debian/opt/cloud/bin/get_template_version.sh similarity index 100% rename from systemvm/patches/debian/opt/cloud/bin/get_template_version.sh rename to systemvm/debian/opt/cloud/bin/get_template_version.sh diff --git a/systemvm/patches/debian/opt/cloud/bin/ilb.sh b/systemvm/debian/opt/cloud/bin/ilb.sh similarity index 100% rename from systemvm/patches/debian/opt/cloud/bin/ilb.sh rename to systemvm/debian/opt/cloud/bin/ilb.sh diff --git a/systemvm/patches/debian/opt/cloud/bin/ipassoc.sh b/systemvm/debian/opt/cloud/bin/ipassoc.sh similarity index 100% rename from systemvm/patches/debian/opt/cloud/bin/ipassoc.sh rename to systemvm/debian/opt/cloud/bin/ipassoc.sh diff --git a/systemvm/patches/debian/opt/cloud/bin/ipsectunnel.sh b/systemvm/debian/opt/cloud/bin/ipsectunnel.sh similarity index 100% rename from systemvm/patches/debian/opt/cloud/bin/ipsectunnel.sh rename to systemvm/debian/opt/cloud/bin/ipsectunnel.sh diff --git a/systemvm/patches/debian/opt/cloud/bin/line_edit.py b/systemvm/debian/opt/cloud/bin/line_edit.py similarity index 100% rename from systemvm/patches/debian/opt/cloud/bin/line_edit.py rename to systemvm/debian/opt/cloud/bin/line_edit.py diff --git a/systemvm/patches/debian/opt/cloud/bin/loadbalancer.sh b/systemvm/debian/opt/cloud/bin/loadbalancer.sh similarity index 100% rename from systemvm/patches/debian/opt/cloud/bin/loadbalancer.sh rename to systemvm/debian/opt/cloud/bin/loadbalancer.sh diff --git a/systemvm/patches/debian/opt/cloud/bin/master.py b/systemvm/debian/opt/cloud/bin/master.py similarity index 95% rename from systemvm/patches/debian/opt/cloud/bin/master.py rename to systemvm/debian/opt/cloud/bin/master.py index 3d1dcd7ef9a..26de8b93bb3 100755 --- a/systemvm/patches/debian/opt/cloud/bin/master.py +++ b/systemvm/debian/opt/cloud/bin/master.py @@ -42,7 +42,7 @@ logging.basicConfig(filename=config.get_logger(), format=config.get_format()) config.cmdline() cl = CsCmdLine("cmdline", config) -#Update the configuration to set state as backup and let keepalived decide who the real Master is! +# Update the configuration to set state as backup and let keepalived decide who the real Master is! cl.set_master_state(False) cl.save() diff --git a/systemvm/patches/debian/opt/cloud/bin/merge.py b/systemvm/debian/opt/cloud/bin/merge.py similarity index 98% rename from systemvm/patches/debian/opt/cloud/bin/merge.py rename to systemvm/debian/opt/cloud/bin/merge.py index ef20d0b32cb..0aaa3401f79 100755 --- a/systemvm/patches/debian/opt/cloud/bin/merge.py +++ b/systemvm/debian/opt/cloud/bin/merge.py @@ -271,6 +271,7 @@ class updateDataBag: dbag = cs_ip.merge(dbag, ip) return dbag + class QueueFile: fileName = '' @@ -284,15 +285,15 @@ class QueueFile: self.type = self.data["type"] updateDataBag(self) return - filename = '{cache_location}/{json_file}'.format(cache_location = self.configCache, json_file = self.fileName) + filename = '{cache_location}/{json_file}'.format(cache_location=self.configCache, json_file=self.fileName) try: handle = open(filename) except IOError as exception: - error_message = ("Exception occurred with the following exception error '{error}'. Could not open '{file}'. " - "It seems that the file has already been moved.".format(error = exception, file = filename)) + error_message = ("Exception occurred with the following exception error '{error}'. Could not open '{filename}'. " + "It seems that the file has already been moved.".format(error=exception, filename=filename)) logging.error(error_message) else: - logging.info("Continuing with the processing of file '{file}'".format(file = filename)) + logging.info("Continuing with the processing of file '{filename}'".format(filename=filename)) self.data = json.load(handle) self.type = self.data["type"] @@ -331,7 +332,6 @@ class QueueFile: class PrivateGatewayHack: - @classmethod def update_network_type_for_privategateway(cls, dbag, data): ip = data['router_guest_ip'] if 'router_guest_ip' in data.keys() else data['public_ip'] @@ -347,12 +347,10 @@ class PrivateGatewayHack: logging.debug("Not updating nw_type for ip %s because has_private_gw_ip = %s and private_gw_matches = %s " % (ip, has_private_gw_ip, private_gw_matches)) return data - @classmethod def if_config_has_privategateway(cls, dbag): return 'privategateway' in dbag['config'].keys() and dbag['config']['privategateway'] != "None" - @classmethod def ip_matches_private_gateway_ip(cls, ip, private_gateway_ip): new_ip_matches_private_gateway_ip = False @@ -360,7 +358,6 @@ class PrivateGatewayHack: new_ip_matches_private_gateway_ip = True return new_ip_matches_private_gateway_ip - @classmethod def load_inital_data(cls): initial_data_bag = DataBag() diff --git a/systemvm/patches/debian/opt/cloud/bin/monitor_service.sh b/systemvm/debian/opt/cloud/bin/monitor_service.sh similarity index 100% rename from systemvm/patches/debian/opt/cloud/bin/monitor_service.sh rename to systemvm/debian/opt/cloud/bin/monitor_service.sh diff --git a/systemvm/patches/debian/opt/cloud/bin/netusage.sh b/systemvm/debian/opt/cloud/bin/netusage.sh similarity index 100% rename from systemvm/patches/debian/opt/cloud/bin/netusage.sh rename to systemvm/debian/opt/cloud/bin/netusage.sh diff --git a/systemvm/patches/debian/opt/cloud/bin/passwd_server b/systemvm/debian/opt/cloud/bin/passwd_server similarity index 100% rename from systemvm/patches/debian/opt/cloud/bin/passwd_server rename to systemvm/debian/opt/cloud/bin/passwd_server diff --git a/systemvm/patches/debian/opt/cloud/bin/passwd_server_ip b/systemvm/debian/opt/cloud/bin/passwd_server_ip similarity index 100% rename from systemvm/patches/debian/opt/cloud/bin/passwd_server_ip rename to systemvm/debian/opt/cloud/bin/passwd_server_ip diff --git a/systemvm/patches/debian/opt/cloud/bin/passwd_server_ip.py b/systemvm/debian/opt/cloud/bin/passwd_server_ip.py similarity index 100% rename from systemvm/patches/debian/opt/cloud/bin/passwd_server_ip.py rename to systemvm/debian/opt/cloud/bin/passwd_server_ip.py diff --git a/systemvm/patches/debian/opt/cloud/bin/prepare_pxe.sh b/systemvm/debian/opt/cloud/bin/prepare_pxe.sh similarity index 100% rename from systemvm/patches/debian/opt/cloud/bin/prepare_pxe.sh rename to systemvm/debian/opt/cloud/bin/prepare_pxe.sh diff --git a/systemvm/patches/debian/opt/cloud/bin/savepassword.sh b/systemvm/debian/opt/cloud/bin/savepassword.sh similarity index 100% rename from systemvm/patches/debian/opt/cloud/bin/savepassword.sh rename to systemvm/debian/opt/cloud/bin/savepassword.sh diff --git a/systemvm/patches/debian/opt/cloud/bin/set_redundant.py b/systemvm/debian/opt/cloud/bin/set_redundant.py similarity index 91% rename from systemvm/patches/debian/opt/cloud/bin/set_redundant.py rename to systemvm/debian/opt/cloud/bin/set_redundant.py index 7a8dad026d2..4d3ca242d41 100755 --- a/systemvm/patches/debian/opt/cloud/bin/set_redundant.py +++ b/systemvm/debian/opt/cloud/bin/set_redundant.py @@ -37,11 +37,10 @@ config = CsConfig() logging.basicConfig(filename=config.get_logger(), level=config.get_level(), format=config.get_format()) -config.set_cl() if options.enable: - config.get_cmdline().set_redundant("true") + config.cmdline().set_redundant("true") if options.disable: - config.get_cmdline().set_redundant("false") + config.cmdline().set_redundant("false") -config.get_cmdline().save() +config.cmdline().save() diff --git a/systemvm/patches/debian/etc/init.d/cloud-early-config b/systemvm/debian/opt/cloud/bin/setup/cloud-early-config similarity index 80% rename from systemvm/patches/debian/etc/init.d/cloud-early-config rename to systemvm/debian/opt/cloud/bin/setup/cloud-early-config index 9964f903754..cec2d3cd2b5 100755 --- a/systemvm/patches/debian/etc/init.d/cloud-early-config +++ b/systemvm/debian/opt/cloud/bin/setup/cloud-early-config @@ -1,12 +1,4 @@ #!/bin/bash -### BEGIN INIT INFO -# Provides: cloud-early-config -# Required-Start: $local_fs -# Required-Stop: $local_fs -# Default-Start: S -# Default-Stop: 0 6 -# Short-Description: configures systemvm using cmdline -### END INIT INFO # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -30,6 +22,7 @@ PATH="/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin" CMDLINE=/var/cache/cloud/cmdline # Clear boot up flag, it would be created by rc.local after boot up done +mkdir -p /var/cache/cloud rm -f /var/cache/cloud/boot_up_done [ -x /sbin/ifup ] || exit 0 @@ -62,12 +55,6 @@ config_guest() { # Configure hot-plug modprobe acpiphp || true modprobe pci_hotplug || true - # Configure serial console FIXME: remove during build we enable it? - #sed -i -e "/^serial.*/d" /boot/grub/grub.conf - #sed -i -e "/^terminal.*/d" /boot/grub/grub.conf - #sed -i -e "/^default.*/a\serial --unit=0 --speed=115200 --parity=no --stop=1" /boot/grub/grub.conf - #sed -i -e "/^serial.*/a\terminal --timeout=0 serial console" /boot/grub/grub.conf - #sed -i -e "s/\(^kernel.* ro\) \(console.*\)/\1 console=tty0 console=ttyS0,115200n8/" /boot/grub/grub.conf sed -i -e "/^s0:2345:respawn.*/d" /etc/inittab sed -i -e "/6:23:respawn/a\s0:2345:respawn:/sbin/getty -L 115200 ttyS0 vt102" /etc/inittab fi @@ -192,10 +179,10 @@ patch() { tar xzf $patchfile -C / echo ${newmd5} > ${md5file} log_it "Patched scripts using $patchfile" - fi - log_it "Patching cloud service" - /opt/cloud/bin/setup/patchsystemvm.sh $PATCH_MOUNT $TYPE + log_it "Patching cloud service" + /opt/cloud/bin/setup/patchsystemvm.sh $PATCH_MOUNT $TYPE + fi [ -f $privkey ] && cp -f $privkey /root/.ssh/ && chmod go-rwx /root/.ssh/authorized_keys umount $PATCH_MOUNT @@ -209,6 +196,8 @@ patch() { } start() { + log_it "Executing cloud-early-config" + # Clear /tmp for file lock rm -f /tmp/*.lock rm -f /tmp/rrouter_bumped @@ -236,41 +225,7 @@ start() { log_it "Finished setting up systemvm" - return 0 + exit 0 } -case "$1" in - start) - log_action_begin_msg "Executing cloud-early-config" - log_it "Executing cloud-early-config" - if start; then - log_action_end_msg $? - else - log_action_end_msg $? - fi - ;; - - stop) - log_action_begin_msg "Stopping cloud-early-config" - #Override old system's interface setting - setup_default; - log_action_end_msg 0 - ;; - - force-reload|restart) - log_warning_msg "Running $0 is deprecated because it may not enable again some interfaces" - log_action_begin_msg "Executing cloud-early-config" - if start; then - log_action_end_msg $? - else - log_action_end_msg $? - fi - ;; - - *) - echo "Usage: /etc/init.d/cloud-early-config {start|stop}" - exit 1 - ;; -esac - -exit 0 +start diff --git a/systemvm/patches/debian/opt/cloud/bin/setup/common.sh b/systemvm/debian/opt/cloud/bin/setup/common.sh similarity index 98% rename from systemvm/patches/debian/opt/cloud/bin/setup/common.sh rename to systemvm/debian/opt/cloud/bin/setup/common.sh index e476745a4e1..e6a4417f55f 100755 --- a/systemvm/patches/debian/opt/cloud/bin/setup/common.sh +++ b/systemvm/debian/opt/cloud/bin/setup/common.sh @@ -471,7 +471,6 @@ setup_sshd(){ local eth=$2 [ -f /etc/ssh/sshd_config ] && sed -i -e "s/^[#]*ListenAddress.*$/ListenAddress $ip/" /etc/ssh/sshd_config sed -i "/3922/s/eth./$eth/" /etc/iptables/rules.v4 - sed -i "/3922/s/eth./$eth/" /etc/iptables/rules } setup_vpc_apache2() { @@ -596,18 +595,16 @@ routing_svcs() { echo "cloud nfs-common portmap" > /var/cache/cloud/disabled_svcs if [ $RROUTER -eq 1 ] then - systemctl disable --now cloud-passwd-srvr systemctl disable --now dnsmasq systemctl enable conntrackd systemctl enable keepalived echo "keepalived conntrackd" >> /var/cache/cloud/enabled_svcs - echo "dnsmasq cloud-passwd-srvr" >> /var/cache/cloud/disabled_svcs + echo "dnsmasq" >> /var/cache/cloud/disabled_svcs else systemctl disable --now conntrackd systemctl disable --now keepalived - systemctl enable cloud-passwd-srvr systemctl enable dnsmasq - echo "dnsmasq cloud-passwd-srvr " >> /var/cache/cloud/enabled_svcs + echo "dnsmasq" >> /var/cache/cloud/enabled_svcs echo "keepalived conntrackd " >> /var/cache/cloud/disabled_svcs fi } @@ -636,6 +633,7 @@ setup_redundant_router() { then sed -i "s/advert_int 1/advert_int $ADVERT_INT/g" /etc/keepalived/keepalived.conf fi + chmod -x /etc/keepalived/keepalived.conf # Seed conntrackd cp /opt/cloud/templates/conntrackd.conf.templ /etc/conntrackd/conntrackd.conf diff --git a/systemvm/patches/debian/opt/cloud/bin/setup/consoleproxy.sh b/systemvm/debian/opt/cloud/bin/setup/consoleproxy.sh similarity index 91% rename from systemvm/patches/debian/opt/cloud/bin/setup/consoleproxy.sh rename to systemvm/debian/opt/cloud/bin/setup/consoleproxy.sh index 36c1fa41115..a3b2797f850 100755 --- a/systemvm/patches/debian/opt/cloud/bin/setup/consoleproxy.sh +++ b/systemvm/debian/opt/cloud/bin/setup/consoleproxy.sh @@ -20,7 +20,7 @@ consoleproxy_svcs() { echo "cloud ssh" > /var/cache/cloud/enabled_svcs - echo "cloud-passwd-srvr haproxy dnsmasq apache2 nfs-common portmap" > /var/cache/cloud/disabled_svcs + echo "haproxy dnsmasq apache2 nfs-common portmap" > /var/cache/cloud/disabled_svcs mkdir -p /var/log/cloud } @@ -36,7 +36,6 @@ setup_console_proxy() { log_it "Applying iptables rules" cp /etc/iptables/iptables-consoleproxy /etc/iptables/rules.v4 - cp /etc/iptables/iptables-consoleproxy /etc/iptables/rules log_it "Configuring sshd" local hyp=$HYPERVISOR diff --git a/systemvm/patches/debian/opt/cloud/bin/setup/default.sh b/systemvm/debian/opt/cloud/bin/setup/default.sh similarity index 100% rename from systemvm/patches/debian/opt/cloud/bin/setup/default.sh rename to systemvm/debian/opt/cloud/bin/setup/default.sh diff --git a/systemvm/patches/debian/opt/cloud/bin/setup/dhcpsrvr.sh b/systemvm/debian/opt/cloud/bin/setup/dhcpsrvr.sh similarity index 80% rename from systemvm/patches/debian/opt/cloud/bin/setup/dhcpsrvr.sh rename to systemvm/debian/opt/cloud/bin/setup/dhcpsrvr.sh index 1e28ab45857..467bb56171a 100755 --- a/systemvm/patches/debian/opt/cloud/bin/setup/dhcpsrvr.sh +++ b/systemvm/debian/opt/cloud/bin/setup/dhcpsrvr.sh @@ -19,7 +19,7 @@ . /opt/cloud/bin/setup/common.sh dhcpsrvr_svcs() { - echo "ssh dnsmasq cloud-passwd-srvr apache2" > /var/cache/cloud/enabled_svcs + echo "ssh dnsmasq apache2" > /var/cache/cloud/enabled_svcs echo "cloud nfs-common conntrackd keepalived haproxy portmap" > /var/cache/cloud/disabled_svcs } @@ -37,13 +37,10 @@ setup_dhcpsrvr() { enable_fwding 0 cp /etc/iptables/iptables-router /etc/iptables/rules.v4 - cp /etc/iptables/iptables-router /etc/iptables/rules #Only allow DNS service for current network sed -i "s/-A INPUT -i eth0 -p udp -m udp --dport 53 -j ACCEPT/-A INPUT -i eth0 -p udp -m udp --dport 53 -s $DHCP_RANGE\/$CIDR_SIZE -j ACCEPT/g" /etc/iptables/rules.v4 - sed -i "s/-A INPUT -i eth0 -p udp -m udp --dport 53 -j ACCEPT/-A INPUT -i eth0 -p udp -m udp --dport 53 -s $DHCP_RANGE\/$CIDR_SIZE -j ACCEPT/g" /etc/iptables/rules sed -i "s/-A INPUT -i eth0 -p tcp -m tcp --dport 53 -j ACCEPT/-A INPUT -i eth0 -p tcp -m tcp --dport 53 -s $DHCP_RANGE\/$CIDR_SIZE -j ACCEPT/g" /etc/iptables/rules.v4 - sed -i "s/-A INPUT -i eth0 -p tcp -m tcp --dport 53 -j ACCEPT/-A INPUT -i eth0 -p tcp -m tcp --dport 53 -s $DHCP_RANGE\/$CIDR_SIZE -j ACCEPT/g" /etc/iptables/rules if [ "$SSHONGUEST" == "true" ] then diff --git a/systemvm/patches/debian/opt/cloud/bin/setup/elbvm.sh b/systemvm/debian/opt/cloud/bin/setup/elbvm.sh similarity index 89% rename from systemvm/patches/debian/opt/cloud/bin/setup/elbvm.sh rename to systemvm/debian/opt/cloud/bin/setup/elbvm.sh index 23a2607ff99..600fb0d0aae 100755 --- a/systemvm/patches/debian/opt/cloud/bin/setup/elbvm.sh +++ b/systemvm/debian/opt/cloud/bin/setup/elbvm.sh @@ -20,7 +20,7 @@ elbvm_svcs() { echo "ssh haproxy" > /var/cache/cloud/enabled_svcs - echo "cloud dnsmasq conntrackd keepalived cloud-passwd-srvr apache2 nfs-common portmap" > /var/cache/cloud/disabled_svcs + echo "cloud dnsmasq conntrackd keepalived apache2 nfs-common portmap" > /var/cache/cloud/disabled_svcs } setup_elbvm() { @@ -32,7 +32,6 @@ setup_elbvm() { echo "$public_ip $NAME" >> /etc/hosts cp /etc/iptables/iptables-elbvm /etc/iptables/rules.v4 - cp /etc/iptables/iptables-elbvm /etc/iptables/rules if [ "$SSHONGUEST" == "true" ] then setup_sshd $ETH0_IP "eth0" diff --git a/systemvm/patches/debian/opt/cloud/bin/setup/ilbvm.sh b/systemvm/debian/opt/cloud/bin/setup/ilbvm.sh similarity index 88% rename from systemvm/patches/debian/opt/cloud/bin/setup/ilbvm.sh rename to systemvm/debian/opt/cloud/bin/setup/ilbvm.sh index 4cd599f59d7..58a711c8df5 100755 --- a/systemvm/patches/debian/opt/cloud/bin/setup/ilbvm.sh +++ b/systemvm/debian/opt/cloud/bin/setup/ilbvm.sh @@ -20,7 +20,7 @@ ilbvm_svcs() { echo "ssh haproxy" > /var/cache/cloud/enabled_svcs - echo "cloud dnsmasq conntrackd keepalived cloud-passwd-srvr apache2 nfs-common portmap" > /var/cache/cloud/disabled_svcs + echo "cloud dnsmasq conntrackd keepalived apache2 nfs-common portmap" > /var/cache/cloud/disabled_svcs } setup_ilbvm() { @@ -32,7 +32,6 @@ setup_ilbvm() { echo "$ETH0_IP $NAME" >> /etc/hosts cp /etc/iptables/iptables-ilbvm /etc/iptables/rules.v4 - cp /etc/iptables/iptables-ilbvm /etc/iptables/rules setup_sshd $ETH1_IP "eth1" enable_fwding 0 diff --git a/systemvm/patches/debian/opt/cloud/bin/setup/patchsystemvm.sh b/systemvm/debian/opt/cloud/bin/setup/patchsystemvm.sh similarity index 99% rename from systemvm/patches/debian/opt/cloud/bin/setup/patchsystemvm.sh rename to systemvm/debian/opt/cloud/bin/setup/patchsystemvm.sh index d09cc03c949..8642945bece 100755 --- a/systemvm/patches/debian/opt/cloud/bin/setup/patchsystemvm.sh +++ b/systemvm/debian/opt/cloud/bin/setup/patchsystemvm.sh @@ -1,4 +1,4 @@ -#/bin/bash +#!/bin/bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/systemvm/debian/opt/cloud/bin/setup/postinit.sh b/systemvm/debian/opt/cloud/bin/setup/postinit.sh new file mode 100755 index 00000000000..efd1b4ba206 --- /dev/null +++ b/systemvm/debian/opt/cloud/bin/setup/postinit.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This scripts before ssh.service but after cloud-early-config + +CMDLINE=/var/cache/cloud/cmdline +for str in $(cat $CMDLINE) + do + KEY=$(echo $str | cut -d= -f1) + VALUE=$(echo $str | cut -d= -f2) + case $KEY in + type) + export TYPE=$VALUE + ;; + *) + ;; + esac +done + +chmod -x /etc/systemd/system/cloud*.service +systemctl daemon-reload + +if [ "$TYPE" == "router" ] || [ "$TYPE" == "vpcrouter" ] || [ "$TYPE" == "dhcpsrvr" ] +then + if [ -x /opt/cloud/bin/update_config.py ] + then + /opt/cloud/bin/update_config.py cmd_line.json + logger -t cloud "Updated config: cmd_line.json" + fi +fi + +if [ "$TYPE" == "router" ] +then + python /opt/cloud/bin/baremetal-vr.py & + logger -t cloud "Started baremetal-vr service" +fi diff --git a/systemvm/patches/debian/opt/cloud/bin/setup/router.sh b/systemvm/debian/opt/cloud/bin/setup/router.sh similarity index 88% rename from systemvm/patches/debian/opt/cloud/bin/setup/router.sh rename to systemvm/debian/opt/cloud/bin/setup/router.sh index d9fadfef4b1..3bd42240ec7 100755 --- a/systemvm/patches/debian/opt/cloud/bin/setup/router.sh +++ b/systemvm/debian/opt/cloud/bin/setup/router.sh @@ -83,15 +83,11 @@ setup_router() { enable_fwding 1 enable_rpsrfs 1 cp /etc/iptables/iptables-router /etc/iptables/rules.v4 - #for old templates - cp /etc/iptables/iptables-router /etc/iptables/rules setup_sshd $ETH1_IP "eth1" #Only allow DNS service for current network sed -i "s/-A INPUT -i eth0 -p udp -m udp --dport 53 -j ACCEPT/-A INPUT -i eth0 -p udp -m udp --dport 53 -s $DHCP_RANGE\/$CIDR_SIZE -j ACCEPT/g" /etc/iptables/rules.v4 - sed -i "s/-A INPUT -i eth0 -p udp -m udp --dport 53 -j ACCEPT/-A INPUT -i eth0 -p udp -m udp --dport 53 -s $DHCP_RANGE\/$CIDR_SIZE -j ACCEPT/g" /etc/iptables/rules sed -i "s/-A INPUT -i eth0 -p tcp -m tcp --dport 53 -j ACCEPT/-A INPUT -i eth0 -p tcp -m tcp --dport 53 -s $DHCP_RANGE\/$CIDR_SIZE -j ACCEPT/g" /etc/iptables/rules.v4 - sed -i "s/-A INPUT -i eth0 -p tcp -m tcp --dport 53 -j ACCEPT/-A INPUT -i eth0 -p tcp -m tcp --dport 53 -s $DHCP_RANGE\/$CIDR_SIZE -j ACCEPT/g" /etc/iptables/rules #setup hourly logrotate mv -n /etc/cron.daily/logrotate /etc/cron.hourly 2>&1 diff --git a/systemvm/patches/debian/opt/cloud/bin/setup/secstorage.sh b/systemvm/debian/opt/cloud/bin/setup/secstorage.sh similarity index 94% rename from systemvm/patches/debian/opt/cloud/bin/setup/secstorage.sh rename to systemvm/debian/opt/cloud/bin/setup/secstorage.sh index 327353c0ae7..b890d77d752 100755 --- a/systemvm/patches/debian/opt/cloud/bin/setup/secstorage.sh +++ b/systemvm/debian/opt/cloud/bin/setup/secstorage.sh @@ -20,7 +20,7 @@ secstorage_svcs() { echo "apache2 cloud ssh nfs-common portmap" > /var/cache/cloud/enabled_svcs - echo "cloud-passwd-srvr conntrackd keepalived haproxy dnsmasq" > /var/cache/cloud/disabled_svcs + echo "conntrackd keepalived haproxy dnsmasq" > /var/cache/cloud/disabled_svcs mkdir -p /var/log/cloud } @@ -39,7 +39,6 @@ setup_secstorage() { log_it "Applying iptables rules" cp /etc/iptables/iptables-secstorage /etc/iptables/rules.v4 - cp /etc/iptables/iptables-secstorage /etc/iptables/rules log_it "Configuring sshd" local hyp=$HYPERVISOR diff --git a/systemvm/patches/debian/opt/cloud/bin/setup/vpcrouter.sh b/systemvm/debian/opt/cloud/bin/setup/vpcrouter.sh similarity index 98% rename from systemvm/patches/debian/opt/cloud/bin/setup/vpcrouter.sh rename to systemvm/debian/opt/cloud/bin/setup/vpcrouter.sh index bd1181fa805..5ed09c7031c 100755 --- a/systemvm/patches/debian/opt/cloud/bin/setup/vpcrouter.sh +++ b/systemvm/debian/opt/cloud/bin/setup/vpcrouter.sh @@ -93,7 +93,6 @@ EOF disable_rpfilter enable_fwding 1 cp /etc/iptables/iptables-vpcrouter /etc/iptables/rules.v4 - cp /etc/iptables/iptables-vpcrouter /etc/iptables/rules setup_sshd $ETH0_IP "eth0" cp /etc/vpcdnsmasq.conf /etc/dnsmasq.conf cp /etc/cloud-nic.rules /etc/udev/rules.d/cloud-nic.rules diff --git a/systemvm/patches/debian/opt/cloud/bin/update_config.py b/systemvm/debian/opt/cloud/bin/update_config.py similarity index 92% rename from systemvm/patches/debian/opt/cloud/bin/update_config.py rename to systemvm/debian/opt/cloud/bin/update_config.py index e36980a4365..a603f47e3ae 100755 --- a/systemvm/patches/debian/opt/cloud/bin/update_config.py +++ b/systemvm/debian/opt/cloud/bin/update_config.py @@ -26,8 +26,6 @@ import os.path import configure import json -OCCURRENCES = 1 - logging.basicConfig(filename='/var/log/cloud.log', level=logging.INFO, format='%(asctime)s %(filename)s %(funcName)s:%(lineno)d %(message)s') # first commandline argument should be the file to process @@ -37,10 +35,10 @@ if (len(sys.argv) != 2): # FIXME we should get this location from a configuration class jsonPath = "/var/cache/cloud/%s" -jsonCmdConfigPath = jsonPath % sys.argv[1] +jsonFilename = sys.argv[1] +jsonConfigFile = jsonPath % jsonFilename currentGuestNetConfig = "/etc/cloudstack/guestnetwork.json" - # If the command line json file is unprocessed process it # This is important or, the control interfaces will get deleted! if os.path.isfile(jsonPath % "cmd_line.json"): @@ -82,7 +80,7 @@ def is_guestnet_configured(guestnet_dict, keys): print "[WARN] update_config.py :: Reconfiguring guest network..." return False - file = open(jsonCmdConfigPath) + file = open(jsonConfigFile) new_guestnet_dict = json.load(file) if not new_guestnet_dict['add']: @@ -121,13 +119,13 @@ def is_guestnet_configured(guestnet_dict, keys): return exists -filename = jsonCmdConfigPath -if not (os.path.isfile(filename) and os.access(filename, os.R_OK)): - print "[ERROR] update_config.py :: You are telling me to process %s, but i can't access it" % jsonCmdConfigPath + +if not (os.path.isfile(jsonConfigFile) and os.access(jsonConfigFile, os.R_OK)): + print "[ERROR] update_config.py :: Unable to read and access %s to process it" % jsonConfigFile sys.exit(1) # If the guest network is already configured and have the same IP, do not try to configure it again otherwise it will break -if sys.argv[1] and sys.argv[1].count("guest_network.json") == OCCURRENCES: +if jsonFilename.startswith("guest_network.json"): if os.path.isfile(currentGuestNetConfig): file = open(currentGuestNetConfig) guestnet_dict = json.load(file) diff --git a/systemvm/patches/debian/opt/cloud/bin/vmdata.py b/systemvm/debian/opt/cloud/bin/vmdata.py similarity index 99% rename from systemvm/patches/debian/opt/cloud/bin/vmdata.py rename to systemvm/debian/opt/cloud/bin/vmdata.py index b9127a1b998..65f52789771 100755 --- a/systemvm/patches/debian/opt/cloud/bin/vmdata.py +++ b/systemvm/debian/opt/cloud/bin/vmdata.py @@ -159,7 +159,7 @@ def exflock(file): def unflock(file): try: flock(file, LOCK_UN) - except IOError: + except IOError as e: print "failed to unlock file" + file.name + " due to : " + e.strerror sys.exit(1) return True diff --git a/systemvm/patches/debian/opt/cloud/bin/vpc_func.sh b/systemvm/debian/opt/cloud/bin/vpc_func.sh similarity index 100% rename from systemvm/patches/debian/opt/cloud/bin/vpc_func.sh rename to systemvm/debian/opt/cloud/bin/vpc_func.sh diff --git a/systemvm/patches/debian/opt/cloud/bin/vpc_netusage.sh b/systemvm/debian/opt/cloud/bin/vpc_netusage.sh similarity index 100% rename from systemvm/patches/debian/opt/cloud/bin/vpc_netusage.sh rename to systemvm/debian/opt/cloud/bin/vpc_netusage.sh diff --git a/systemvm/patches/debian/opt/cloud/bin/vpc_passwd_server b/systemvm/debian/opt/cloud/bin/vpc_passwd_server similarity index 100% rename from systemvm/patches/debian/opt/cloud/bin/vpc_passwd_server rename to systemvm/debian/opt/cloud/bin/vpc_passwd_server diff --git a/systemvm/patches/debian/opt/cloud/bin/vpc_snat.sh b/systemvm/debian/opt/cloud/bin/vpc_snat.sh similarity index 100% rename from systemvm/patches/debian/opt/cloud/bin/vpc_snat.sh rename to systemvm/debian/opt/cloud/bin/vpc_snat.sh diff --git a/systemvm/patches/debian/opt/cloud/bin/vpc_staticroute.sh b/systemvm/debian/opt/cloud/bin/vpc_staticroute.sh similarity index 100% rename from systemvm/patches/debian/opt/cloud/bin/vpc_staticroute.sh rename to systemvm/debian/opt/cloud/bin/vpc_staticroute.sh diff --git a/systemvm/patches/vpn/opt/cloud/bin/vpn_l2tp.sh b/systemvm/debian/opt/cloud/bin/vpn_l2tp.sh similarity index 100% rename from systemvm/patches/vpn/opt/cloud/bin/vpn_l2tp.sh rename to systemvm/debian/opt/cloud/bin/vpn_l2tp.sh diff --git a/systemvm/patches/debian/opt/cloud/bin/vr_cfg.sh b/systemvm/debian/opt/cloud/bin/vr_cfg.sh similarity index 100% rename from systemvm/patches/debian/opt/cloud/bin/vr_cfg.sh rename to systemvm/debian/opt/cloud/bin/vr_cfg.sh diff --git a/systemvm/patches/debian/opt/cloud/templates/README b/systemvm/debian/opt/cloud/templates/README similarity index 100% rename from systemvm/patches/debian/opt/cloud/templates/README rename to systemvm/debian/opt/cloud/templates/README diff --git a/systemvm/patches/debian/opt/cloud/templates/arping_gateways.sh.templ b/systemvm/debian/opt/cloud/templates/arping_gateways.sh.templ similarity index 100% rename from systemvm/patches/debian/opt/cloud/templates/arping_gateways.sh.templ rename to systemvm/debian/opt/cloud/templates/arping_gateways.sh.templ diff --git a/systemvm/patches/debian/opt/cloud/templates/check_bumpup.sh b/systemvm/debian/opt/cloud/templates/check_bumpup.sh similarity index 100% rename from systemvm/patches/debian/opt/cloud/templates/check_bumpup.sh rename to systemvm/debian/opt/cloud/templates/check_bumpup.sh diff --git a/systemvm/patches/debian/opt/cloud/templates/check_heartbeat.sh.templ b/systemvm/debian/opt/cloud/templates/check_heartbeat.sh.templ similarity index 100% rename from systemvm/patches/debian/opt/cloud/templates/check_heartbeat.sh.templ rename to systemvm/debian/opt/cloud/templates/check_heartbeat.sh.templ diff --git a/systemvm/patches/debian/opt/cloud/templates/checkrouter.sh.templ b/systemvm/debian/opt/cloud/templates/checkrouter.sh.templ similarity index 100% rename from systemvm/patches/debian/opt/cloud/templates/checkrouter.sh.templ rename to systemvm/debian/opt/cloud/templates/checkrouter.sh.templ diff --git a/systemvm/debian/opt/cloud/templates/conntrackd.conf.templ b/systemvm/debian/opt/cloud/templates/conntrackd.conf.templ new file mode 100644 index 00000000000..9443db24743 --- /dev/null +++ b/systemvm/debian/opt/cloud/templates/conntrackd.conf.templ @@ -0,0 +1,419 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Synchronizer settings +# +Sync { + Mode FTFW { + # + # Size of the resend queue (in objects). This is the maximum + # number of objects that can be stored waiting to be confirmed + # via acknoledgment. If you keep this value low, the daemon + # will have less chances to recover state-changes under message + # omission. On the other hand, if you keep this value high, + # the daemon will consume more memory to store dead objects. + # Default is 131072 objects. + # + # ResendQueueSize 131072 + + # + # This parameter allows you to set an initial fixed timeout + # for the committed entries when this node goes from backup + # to primary. This mechanism provides a way to purge entries + # that were not recovered appropriately after the specified + # fixed timeout. If you set a low value, TCP entries in + # Established states with no traffic may hang. For example, + # an SSH connection without KeepAlive enabled. If not set, + # the daemon uses an approximate timeout value calculation + # mechanism. By default, this option is not set. + # + # CommitTimeout 180 + + # + # If the firewall replica goes from primary to backup, + # the conntrackd -t command is invoked in the script. + # This command schedules a flush of the table in N seconds. + # This is useful to purge the connection tracking table of + # zombie entries and avoid clashes with old entries if you + # trigger several consecutive hand-overs. Default is 60 seconds. + # + # PurgeTimeout 60 + + # Set the acknowledgement window size. If you decrease this + # value, the number of acknowlegdments increases. More + # acknowledgments means more overhead as conntrackd has to + # handle more control messages. On the other hand, if you + # increase this value, the resend queue gets more populated. + # This results in more overhead in the queue releasing. + # The following value is based on some practical experiments + # measuring the cycles spent by the acknowledgment handling + # with oprofile. If not set, default window size is 300. + # + # ACKWindowSize 300 + + # + # This clause allows you to disable the external cache. Thus, + # the state entries are directly injected into the kernel + # conntrack table. As a result, you save memory in user-space + # but you consume slots in the kernel conntrack table for + # backup state entries. Moreover, disabling the external cache + # means more CPU consumption. You need a Linux kernel + # >= 2.6.29 to use this feature. By default, this clause is + # set off. If you are installing conntrackd for first time, + # please read the user manual and I encourage you to consider + # using the fail-over scripts instead of enabling this option! + # + # DisableExternalCache Off + } + + # + # Multicast IP and interface where messages are + # broadcasted (dedicated link). IMPORTANT: Make sure + # that iptables accepts traffic for destination + # 225.0.0.50, eg: + # + # iptables -I INPUT -d 225.0.0.50 -j ACCEPT + # iptables -I OUTPUT -d 225.0.0.50 -j ACCEPT + # + Multicast { + # + # Multicast address: The address that you use as destination + # in the synchronization messages. You do not have to add + # this IP to any of your existing interfaces. If any doubt, + # do not modify this value. + # + IPv4_address 225.0.0.50 + + # + # The multicast group that identifies the cluster. If any + # doubt, do not modify this value. + # + Group 3780 + + # + # IP address of the interface that you are going to use to + # send the synchronization messages. Remember that you must + # use a dedicated link for the synchronization messages. + # + IPv4_interface [LINK_IP] + + # + # The name of the interface that you are going to use to + # send the synchronization messages. + # + Interface [LINK_IF] + + # The multicast sender uses a buffer to enqueue the packets + # that are going to be transmitted. The default size of this + # socket buffer is available at /proc/sys/net/core/wmem_default. + # This value determines the chances to have an overrun in the + # sender queue. The overrun results packet loss, thus, losing + # state information that would have to be retransmitted. If you + # notice some packet loss, you may want to increase the size + # of the sender buffer. The default size is usually around + # ~100 KBytes which is fairly small for busy firewalls. + # + SndSocketBuffer 1249280 + + # The multicast receiver uses a buffer to enqueue the packets + # that the socket is pending to handle. The default size of this + # socket buffer is available at /proc/sys/net/core/rmem_default. + # This value determines the chances to have an overrun in the + # receiver queue. The overrun results packet loss, thus, losing + # state information that would have to be retransmitted. If you + # notice some packet loss, you may want to increase the size of + # the receiver buffer. The default size is usually around + # ~100 KBytes which is fairly small for busy firewalls. + # + RcvSocketBuffer 1249280 + + # + # Enable/Disable message checksumming. This is a good + # property to achieve fault-tolerance. In case of doubt, do + # not modify this value. + # + Checksum on + } + # + # You can specify more than one dedicated link. Thus, if one dedicated + # link fails, conntrackd can fail-over to another. Note that adding + # more than one dedicated link does not mean that state-updates will + # be sent to all of them. There is only one active dedicated link at + # a given moment. The `Default' keyword indicates that this interface + # will be selected as the initial dedicated link. You can have + # up to 4 redundant dedicated links. Note: Use different multicast + # groups for every redundant link. + # + # Multicast Default { + # IPv4_address 225.0.0.51 + # Group 3781 + # IPv4_interface 192.168.100.101 + # Interface eth3 + # # SndSocketBuffer 1249280 + # # RcvSocketBuffer 1249280 + # Checksum on + # } + + # + # You can use Unicast UDP instead of Multicast to propagate events. + # Note that you cannot use unicast UDP and Multicast at the same + # time, you can only select one. + # + # UDP { + # + # UDP address that this firewall uses to listen to events. + # + # IPv4_address 192.168.2.100 + # + # or you may want to use an IPv6 address: + # + # IPv6_address fe80::215:58ff:fe28:5a27 + + # + # Destination UDP address that receives events, ie. the other + # firewall's dedicated link address. + # + # IPv4_Destination_Address 192.168.2.101 + # + # or you may want to use an IPv6 address: + # + # IPv6_Destination_Address fe80::2d0:59ff:fe2a:775c + + # + # UDP port used + # + # Port 3780 + + # + # The name of the interface that you are going to use to + # send the synchronization messages. + # + # Interface eth2 + + # + # The sender socket buffer size + # + # SndSocketBuffer 1249280 + + # + # The receiver socket buffer size + # + # RcvSocketBuffer 1249280 + + # + # Enable/Disable message checksumming. + # + # Checksum on + # } + +} + +# +# General settings +# +General { + # + # Set the nice value of the daemon, this value goes from -20 + # (most favorable scheduling) to 19 (least favorable). Using a + # very low value reduces the chances to lose state-change events. + # Default is 0 but this example file sets it to most favourable + # scheduling as this is generally a good idea. See man nice(1) for + # more information. + # + Nice -20 + + # + # Select a different scheduler for the daemon, you can select between + # RR and FIFO and the process priority (minimum is 0, maximum is 99). + # See man sched_setscheduler(2) for more information. Using a RT + # scheduler reduces the chances to overrun the Netlink buffer. + # + # Scheduler { + # Type FIFO + # Priority 99 + # } + + # + # Number of buckets in the cache hashtable. The bigger it is, + # the closer it gets to O(1) at the cost of consuming more memory. + # Read some documents about tuning hashtables for further reference. + # + HashSize 32768 + + # + # Maximum number of conntracks, it should be double of: + # $ cat /proc/sys/net/netfilter/nf_conntrack_max + # since the daemon may keep some dead entries cached for possible + # retransmission during state synchronization. + # + HashLimit 131072 + + LogFile on + + # + # Syslog: on, off or a facility name (daemon (default) or local0..7) + # Default: off + # + Syslog on + + # + # Lockfile + # + LockFile /var/lock/conntrackd.lock + + # + # Unix socket configuration + # + UNIX { + Path /var/run/conntrackd.sock + Backlog 20 + } + + SocketBufferSize 2097152 + SocketBufferSizeMaxGrown 8388608 + + # + # Netlink event socket buffer size. If you do not specify this clause, + # the default buffer size value in /proc/net/core/rmem_default is + # used. This default value is usually around 100 Kbytes which is + # fairly small for busy firewalls. This leads to event message dropping + # and high CPU consumption. This example configuration file sets the + # size to 2 MBytes to avoid this sort of problems. + # + NetlinkBufferSize 2097152 + + # + # The daemon doubles the size of the netlink event socket buffer size + # if it detects netlink event message dropping. This clause sets the + # maximum buffer size growth that can be reached. This example file + # sets the size to 8 MBytes. + # + NetlinkBufferSizeMaxGrowth 8388608 + + # + # If the daemon detects that Netlink is dropping state-change events, + # it automatically schedules a resynchronization against the Kernel + # after 30 seconds (default value). Resynchronizations are expensive + # in terms of CPU consumption since the daemon has to get the full + # kernel state-table and purge state-entries that do not exist anymore. + # Be careful of setting a very small value here. You have the following + # choices: On (enabled, use default 30 seconds value), Off (disabled) + # or Value (in seconds, to set a specific amount of time). If not + # specified, the daemon assumes that this option is enabled. + # + # NetlinkOverrunResync On + + # + # If you want reliable event reporting over Netlink, set on this + # option. If you set on this clause, it is a good idea to set off + # NetlinkOverrunResync. This option is off by default and you need + # a Linux kernel >= 2.6.31. + # + # NetlinkEventsReliable Off + + # + # By default, the daemon receives state updates following an + # event-driven model. You can modify this behaviour by switching to + # polling mode with the PollSecs clause. This clause tells conntrackd + # to dump the states in the kernel every N seconds. With regards to + # synchronization mode, the polling mode can only guarantee that + # long-lifetime states are recovered. The main advantage of this method + # is the reduction in the state replication at the cost of reducing the + # chances of recovering connections. + # + # PollSecs 15 + + # + # The daemon prioritizes the handling of state-change events coming + # from the core. With this clause, you can set the maximum number of + # state-change events (those coming from kernel-space) that the daemon + # will handle after which it will handle other events coming from the + # network or userspace. A low value improves interactivity (in terms of + # real-time behaviour) at the cost of extra CPU consumption. + # Default (if not set) is 100. + # + # EventIterationLimit 100 + + # + # Event filtering: This clause allows you to filter certain traffic, + # There are currently three filter-sets: Protocol, Address and + # State. The filter is attached to an action that can be: Accept or + # Ignore. Thus, you can define the event filtering policy of the + # filter-sets in positive or negative logic depending on your needs. + # You can select if conntrackd filters the event messages from + # user-space or kernel-space. The kernel-space event filtering + # saves some CPU cycles by avoiding the copy of the event message + # from kernel-space to user-space. The kernel-space event filtering + # is prefered, however, you require a Linux kernel >= 2.6.29 to + # filter from kernel-space. If you want to select kernel-space + # event filtering, use the keyword 'Kernelspace' instead of + # 'Userspace'. + # + Filter From Userspace { + # + # Accept only certain protocols: You may want to replicate + # the state of flows depending on their layer 4 protocol. + # + Protocol Accept { + TCP + SCTP + DCCP + UDP + ICMP # This requires a Linux kernel >= 2.6.31 + } + + # + # Ignore traffic for a certain set of IP's: Usually all the + # IP assigned to the firewall since local traffic must be + # ignored, only forwarded connections are worth to replicate. + # Note that these values depends on the local IPs that are + # assigned to the firewall. + # + Address Ignore { + IPv4_address 127.0.0.1 # loopback + IPv4_address [IGNORE_IP1] + IPv4_address [IGNORE_IP2] + IPv4_address [IGNORE_IP3] + #IPv4_address 192.168.0.100 # virtual IP 1 + #IPv4_address 192.168.1.100 # virtual IP 2 + #IPv4_address 192.168.0.1 + #IPv4_address 192.168.1.1 + #IPv4_address 192.168.100.100 # dedicated link ip + # + # You can also specify networks in format IP/cidr. + # IPv4_address 192.168.0.0/24 + # + # You can also specify an IPv6 address + # IPv6_address ::1 + } + + # + # Uncomment this line below if you want to filter by flow state. + # This option introduces a trade-off in the replication: it + # reduces CPU consumption at the cost of having lazy backup + # firewall replicas. The existing TCP states are: SYN_SENT, + # SYN_RECV, ESTABLISHED, FIN_WAIT, CLOSE_WAIT, LAST_ACK, + # TIME_WAIT, CLOSED, LISTEN. + # + # State Accept { + # ESTABLISHED CLOSED TIME_WAIT CLOSE_WAIT for TCP + # } + } + + # default debian service unit file is of Type=notify + Systemd on +} diff --git a/systemvm/patches/debian/opt/cloud/templates/heartbeat.sh.templ b/systemvm/debian/opt/cloud/templates/heartbeat.sh.templ similarity index 100% rename from systemvm/patches/debian/opt/cloud/templates/heartbeat.sh.templ rename to systemvm/debian/opt/cloud/templates/heartbeat.sh.templ diff --git a/systemvm/patches/debian/opt/cloud/templates/keepalived.conf.templ b/systemvm/debian/opt/cloud/templates/keepalived.conf.templ similarity index 94% rename from systemvm/patches/debian/opt/cloud/templates/keepalived.conf.templ rename to systemvm/debian/opt/cloud/templates/keepalived.conf.templ index d14611d895a..5434ec5302f 100644 --- a/systemvm/patches/debian/opt/cloud/templates/keepalived.conf.templ +++ b/systemvm/debian/opt/cloud/templates/keepalived.conf.templ @@ -21,11 +21,11 @@ global_defs { vrrp_script heartbeat { script "[RROUTER_BIN_PATH]/heartbeat.sh" - interval 5 + interval 4 } vrrp_instance inside_network { - state EQUAL + state BACKUP interface eth2 virtual_router_id 51 nopreempt @@ -44,7 +44,6 @@ vrrp_instance inside_network { heartbeat } - !That's the correct path of the master.py file. notify_backup "/opt/cloud/bin/master.py --backup" notify_master "/opt/cloud/bin/master.py --master" notify_fault "/opt/cloud/bin/master.py --fault" diff --git a/systemvm/patches/debian/opt/cloud/testdata/README b/systemvm/debian/opt/cloud/testdata/README similarity index 100% rename from systemvm/patches/debian/opt/cloud/testdata/README rename to systemvm/debian/opt/cloud/testdata/README diff --git a/systemvm/patches/debian/opt/cloud/testdata/acl0001.json b/systemvm/debian/opt/cloud/testdata/acl0001.json similarity index 100% rename from systemvm/patches/debian/opt/cloud/testdata/acl0001.json rename to systemvm/debian/opt/cloud/testdata/acl0001.json diff --git a/systemvm/patches/debian/opt/cloud/testdata/dhcp0001.json b/systemvm/debian/opt/cloud/testdata/dhcp0001.json similarity index 100% rename from systemvm/patches/debian/opt/cloud/testdata/dhcp0001.json rename to systemvm/debian/opt/cloud/testdata/dhcp0001.json diff --git a/systemvm/patches/debian/opt/cloud/testdata/gn0001.json b/systemvm/debian/opt/cloud/testdata/gn0001.json similarity index 100% rename from systemvm/patches/debian/opt/cloud/testdata/gn0001.json rename to systemvm/debian/opt/cloud/testdata/gn0001.json diff --git a/systemvm/patches/debian/opt/cloud/testdata/ips0001.json b/systemvm/debian/opt/cloud/testdata/ips0001.json similarity index 100% rename from systemvm/patches/debian/opt/cloud/testdata/ips0001.json rename to systemvm/debian/opt/cloud/testdata/ips0001.json diff --git a/systemvm/patches/debian/opt/cloud/testdata/ips0002.json b/systemvm/debian/opt/cloud/testdata/ips0002.json similarity index 100% rename from systemvm/patches/debian/opt/cloud/testdata/ips0002.json rename to systemvm/debian/opt/cloud/testdata/ips0002.json diff --git a/systemvm/patches/debian/opt/cloud/testdata/ips0003.json b/systemvm/debian/opt/cloud/testdata/ips0003.json similarity index 100% rename from systemvm/patches/debian/opt/cloud/testdata/ips0003.json rename to systemvm/debian/opt/cloud/testdata/ips0003.json diff --git a/systemvm/patches/debian/opt/cloud/testdata/s2s0001.json b/systemvm/debian/opt/cloud/testdata/s2s0001.json similarity index 100% rename from systemvm/patches/debian/opt/cloud/testdata/s2s0001.json rename to systemvm/debian/opt/cloud/testdata/s2s0001.json diff --git a/systemvm/patches/debian/opt/cloud/testdata/test.sh b/systemvm/debian/opt/cloud/testdata/test.sh similarity index 100% rename from systemvm/patches/debian/opt/cloud/testdata/test.sh rename to systemvm/debian/opt/cloud/testdata/test.sh diff --git a/systemvm/patches/debian/opt/cloud/testdata/vmp0001.json b/systemvm/debian/opt/cloud/testdata/vmp0001.json similarity index 100% rename from systemvm/patches/debian/opt/cloud/testdata/vmp0001.json rename to systemvm/debian/opt/cloud/testdata/vmp0001.json diff --git a/systemvm/patches/debian/root/.ssh/authorized_keys b/systemvm/debian/root/.ssh/authorized_keys similarity index 100% rename from systemvm/patches/debian/root/.ssh/authorized_keys rename to systemvm/debian/root/.ssh/authorized_keys diff --git a/systemvm/patches/debian/root/clearUsageRules.sh b/systemvm/debian/root/clearUsageRules.sh similarity index 100% rename from systemvm/patches/debian/root/clearUsageRules.sh rename to systemvm/debian/root/clearUsageRules.sh diff --git a/systemvm/patches/debian/root/func.sh b/systemvm/debian/root/func.sh similarity index 100% rename from systemvm/patches/debian/root/func.sh rename to systemvm/debian/root/func.sh diff --git a/systemvm/patches/debian/root/monitorServices.py b/systemvm/debian/root/monitorServices.py similarity index 100% rename from systemvm/patches/debian/root/monitorServices.py rename to systemvm/debian/root/monitorServices.py diff --git a/systemvm/patches/debian/root/reconfigLB.sh b/systemvm/debian/root/reconfigLB.sh similarity index 100% rename from systemvm/patches/debian/root/reconfigLB.sh rename to systemvm/debian/root/reconfigLB.sh diff --git a/systemvm/patches/debian/var/www/html/latest/.htaccess b/systemvm/debian/var/www/html/latest/.htaccess similarity index 100% rename from systemvm/patches/debian/var/www/html/latest/.htaccess rename to systemvm/debian/var/www/html/latest/.htaccess diff --git a/systemvm/patches/debian/var/www/html/userdata/.htaccess b/systemvm/debian/var/www/html/userdata/.htaccess similarity index 100% rename from systemvm/patches/debian/var/www/html/userdata/.htaccess rename to systemvm/debian/var/www/html/userdata/.htaccess diff --git a/systemvm/patches/debian/etc/init.d/cloud-passwd-srvr b/systemvm/patches/debian/etc/init.d/cloud-passwd-srvr deleted file mode 100755 index ffd0ec8d206..00000000000 --- a/systemvm/patches/debian/etc/init.d/cloud-passwd-srvr +++ /dev/null @@ -1,118 +0,0 @@ -#!/bin/bash -### BEGIN INIT INFO -# Provides: cloud-passwd-srvr -# Required-Start: $local_fs cloud-early-config -# Required-Stop: $local_fs -# Default-Start: 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: Web server that sends passwords to User VMs -### END INIT INFO -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -add_iptables_rules() -{ - #Delete any old iptables rule for port 8080 on eth0 - remove_iptables_rules - - #For all cidrs on eth0 for port 8080 accept only if source is withing that cidr - for cidr in $(ip addr | grep eth0 | grep inet | awk '{print $2}'); - do - count=1 - #Try for 10 times, if it still fails then bail - while [ $count -le 10 ]; - do - (( count++ )) - iptables -A INPUT -i eth0 -p tcp -m state --state NEW -m tcp -s $cidr --dport 8080 -j ACCEPT - if [ `iptables -L INPUT -n -v | grep eth0 | grep 8080 | grep ACCEPT | wc -l` -gt 0 ] - then - break - else - sleep 2 - fi - done - done - echo "Added cloud-passwd-srvr iptables rules" && return 0 -} - -remove_iptables_rules() -{ - #Change the Internal Field Separator so the for loop, loops on lines and not spaces - OIFS="${IFS}" - NIFS=$'\n' - IFS="${NIFS}" - - #Removed all iptable rules for port 8080 on eth0, they were added in start() - for srcdest in `iptables -L -n -v | grep eth0 | grep 8080 | grep ACCEPT | awk '{print "--source "$8" --destination "$9}'`; - do - eval "iptables -D INPUT -i eth0 -p tcp -m state --state NEW -m tcp $srcdest --dport 8080 -j ACCEPT"; - done - - #Restore IFS - IFS="${OIFS}" - - echo "Removed cloud-passwd-srvr iptables rules" && return 0 -} - -start() { - pid=$(getpid) - [ "$pid" != "" ] && echo "Password server is already running (pid=$pid)" && return 0 - add_iptables_rules - nohup bash /opt/cloud/bin/passwd_server & -} - -getpid() { - pid=$(ps -ef | grep passwd_server_ip | grep -v grep | awk '{print $2}') - echo $pid -} - -stop_password_server() { - spid=$(pidof -s python passwd_server_ip.py) - [ "$spid" != "" ] && kill -9 $spid && echo "Killed password server (pid=$spid)" - return 0 -} - -stop () { - stop_password_server - pid=$(getpid) - [ "$pid" != "" ] && kill -9 $pid && remove_iptables_rules && echo "Stopped password server (pid=$pid)" && stop_password_server && return 0 - - echo "Password server is not running" && return 0 -} - -status () { - pid=$(getpid) - [ "$pid" != "" ] && echo "Password server is running (pid=$pid)" && return 0 - echo "Password server is not running" && return 0 -} - -case "$1" in - start) start - ;; - stop) stop - ;; - status) status - ;; - restart) stop - start - ;; - *) echo "Usage: $0 {start|stop|status|restart}" - exit 1 - ;; -esac - -exit 0 diff --git a/systemvm/patches/debian/etc/rc.local b/systemvm/patches/debian/etc/rc.local deleted file mode 100755 index 895c120bddf..00000000000 --- a/systemvm/patches/debian/etc/rc.local +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/bash - -#Licensed to the Apache Software Foundation (ASF) under one -#or more contributor license agreements. See the NOTICE file -#distributed with this work for additional information -#regarding copyright ownership. The ASF licenses this file -#to you under the Apache License, Version 2.0 (the -#"License"); you may not use this file except in compliance -#with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -#Unless required by applicable law or agreed to in writing, -#software distributed under the License is distributed on an -#"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -#KIND, either express or implied. See the License for the -#specific language governing permissions and limitations -#under the License. - -[ ! -f /var/cache/cloud/enabled_svcs ] && touch /var/cache/cloud/enabled_svcs -for svc in $(cat /var/cache/cloud/enabled_svcs) -do - logger -t cloud "Starting $svc" - systemctl enable --now $svc -done - -[ ! -f /var/cache/cloud/disabled_svcs ] && touch /var/cache/cloud/disabled_svcs -for svc in $(cat /var/cache/cloud/disabled_svcs) -do - logger -t cloud "Stopping $svc" - systemctl disable --now $svc -done - - -#Restore the persistent iptables nat, rules and filters for IPv4 and IPv6 if they exist -ipv4="/etc/iptables/router_rules.v4" -if [ -e $ipv4 ] -then - iptables-restore < $ipv4 -fi - -ipv6="/etc/iptables/router_rules.v6" -if [ -e $ipv6 ] -then - iptables-restore < $ipv6 -fi - -date > /var/cache/cloud/boot_up_done -logger -t cloud "Boot up process done" - -CMDLINE=/var/cache/cloud/cmdline -for str in $(cat $CMDLINE) - do - KEY=$(echo $str | cut -d= -f1) - VALUE=$(echo $str | cut -d= -f2) - case $KEY in - type) - export TYPE=$VALUE - ;; - *) - ;; - esac -done - -if [ "$TYPE" == "router" ] -then - python /opt/cloud/bin/baremetal-vr.py & - logger -t cloud "Started baremetal-vr service" -fi diff --git a/systemvm/patches/debian/opt/cloud/templates/conntrackd.conf.templ b/systemvm/patches/debian/opt/cloud/templates/conntrackd.conf.templ deleted file mode 100644 index 3e38af1b7d5..00000000000 --- a/systemvm/patches/debian/opt/cloud/templates/conntrackd.conf.templ +++ /dev/null @@ -1,417 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# Synchronizer settings -# -Sync { - Mode FTFW { - # - # Size of the resend queue (in objects). This is the maximum - # number of objects that can be stored waiting to be confirmed - # via acknoledgment. If you keep this value low, the daemon - # will have less chances to recover state-changes under message - # omission. On the other hand, if you keep this value high, - # the daemon will consume more memory to store dead objects. - # Default is 131072 objects. - # - # ResendQueueSize 131072 - - # - # This parameter allows you to set an initial fixed timeout - # for the committed entries when this node goes from backup - # to primary. This mechanism provides a way to purge entries - # that were not recovered appropriately after the specified - # fixed timeout. If you set a low value, TCP entries in - # Established states with no traffic may hang. For example, - # an SSH connection without KeepAlive enabled. If not set, - # the daemon uses an approximate timeout value calculation - # mechanism. By default, this option is not set. - # - # CommitTimeout 180 - - # - # If the firewall replica goes from primary to backup, - # the conntrackd -t command is invoked in the script. - # This command schedules a flush of the table in N seconds. - # This is useful to purge the connection tracking table of - # zombie entries and avoid clashes with old entries if you - # trigger several consecutive hand-overs. Default is 60 seconds. - # - # PurgeTimeout 60 - - # Set the acknowledgement window size. If you decrease this - # value, the number of acknowlegdments increases. More - # acknowledgments means more overhead as conntrackd has to - # handle more control messages. On the other hand, if you - # increase this value, the resend queue gets more populated. - # This results in more overhead in the queue releasing. - # The following value is based on some practical experiments - # measuring the cycles spent by the acknowledgment handling - # with oprofile. If not set, default window size is 300. - # - # ACKWindowSize 300 - - # - # This clause allows you to disable the external cache. Thus, - # the state entries are directly injected into the kernel - # conntrack table. As a result, you save memory in user-space - # but you consume slots in the kernel conntrack table for - # backup state entries. Moreover, disabling the external cache - # means more CPU consumption. You need a Linux kernel - # >= 2.6.29 to use this feature. By default, this clause is - # set off. If you are installing conntrackd for first time, - # please read the user manual and I encourage you to consider - # using the fail-over scripts instead of enabling this option! - # - # DisableExternalCache Off - } - - # - # Multicast IP and interface where messages are - # broadcasted (dedicated link). IMPORTANT: Make sure - # that iptables accepts traffic for destination - # 225.0.0.50, eg: - # - # iptables -I INPUT -d 225.0.0.50 -j ACCEPT - # iptables -I OUTPUT -d 225.0.0.50 -j ACCEPT - # - Multicast { - # - # Multicast address: The address that you use as destination - # in the synchronization messages. You do not have to add - # this IP to any of your existing interfaces. If any doubt, - # do not modify this value. - # - IPv4_address 225.0.0.50 - - # - # The multicast group that identifies the cluster. If any - # doubt, do not modify this value. - # - Group 3780 - - # - # IP address of the interface that you are going to use to - # send the synchronization messages. Remember that you must - # use a dedicated link for the synchronization messages. - # - IPv4_interface [LINK_IP] - - # - # The name of the interface that you are going to use to - # send the synchronization messages. - # - Interface [LINK_IF] - - # The multicast sender uses a buffer to enqueue the packets - # that are going to be transmitted. The default size of this - # socket buffer is available at /proc/sys/net/core/wmem_default. - # This value determines the chances to have an overrun in the - # sender queue. The overrun results packet loss, thus, losing - # state information that would have to be retransmitted. If you - # notice some packet loss, you may want to increase the size - # of the sender buffer. The default size is usually around - # ~100 KBytes which is fairly small for busy firewalls. - # - SndSocketBuffer 1249280 - - # The multicast receiver uses a buffer to enqueue the packets - # that the socket is pending to handle. The default size of this - # socket buffer is available at /proc/sys/net/core/rmem_default. - # This value determines the chances to have an overrun in the - # receiver queue. The overrun results packet loss, thus, losing - # state information that would have to be retransmitted. If you - # notice some packet loss, you may want to increase the size of - # the receiver buffer. The default size is usually around - # ~100 KBytes which is fairly small for busy firewalls. - # - RcvSocketBuffer 1249280 - - # - # Enable/Disable message checksumming. This is a good - # property to achieve fault-tolerance. In case of doubt, do - # not modify this value. - # - Checksum on - } - # - # You can specify more than one dedicated link. Thus, if one dedicated - # link fails, conntrackd can fail-over to another. Note that adding - # more than one dedicated link does not mean that state-updates will - # be sent to all of them. There is only one active dedicated link at - # a given moment. The `Default' keyword indicates that this interface - # will be selected as the initial dedicated link. You can have - # up to 4 redundant dedicated links. Note: Use different multicast - # groups for every redundant link. - # - # Multicast Default { - # IPv4_address 225.0.0.51 - # Group 3781 - # IPv4_interface 192.168.100.101 - # Interface eth3 - # # SndSocketBuffer 1249280 - # # RcvSocketBuffer 1249280 - # Checksum on - # } - - # - # You can use Unicast UDP instead of Multicast to propagate events. - # Note that you cannot use unicast UDP and Multicast at the same - # time, you can only select one. - # - # UDP { - # - # UDP address that this firewall uses to listen to events. - # - # IPv4_address 192.168.2.100 - # - # or you may want to use an IPv6 address: - # - # IPv6_address fe80::215:58ff:fe28:5a27 - - # - # Destination UDP address that receives events, ie. the other - # firewall's dedicated link address. - # - # IPv4_Destination_Address 192.168.2.101 - # - # or you may want to use an IPv6 address: - # - # IPv6_Destination_Address fe80::2d0:59ff:fe2a:775c - - # - # UDP port used - # - # Port 3780 - - # - # The name of the interface that you are going to use to - # send the synchronization messages. - # - # Interface eth2 - - # - # The sender socket buffer size - # - # SndSocketBuffer 1249280 - - # - # The receiver socket buffer size - # - # RcvSocketBuffer 1249280 - - # - # Enable/Disable message checksumming. - # - # Checksum on - # } - -} - -# -# General settings -# -General { - # - # Set the nice value of the daemon, this value goes from -20 - # (most favorable scheduling) to 19 (least favorable). Using a - # very low value reduces the chances to lose state-change events. - # Default is 0 but this example file sets it to most favourable - # scheduling as this is generally a good idea. See man nice(1) for - # more information. - # - Nice -20 - - # - # Select a different scheduler for the daemon, you can select between - # RR and FIFO and the process priority (minimum is 0, maximum is 99). - # See man sched_setscheduler(2) for more information. Using a RT - # scheduler reduces the chances to overrun the Netlink buffer. - # - # Scheduler { - # Type FIFO - # Priority 99 - # } - - # - # Number of buckets in the cache hashtable. The bigger it is, - # the closer it gets to O(1) at the cost of consuming more memory. - # Read some documents about tuning hashtables for further reference. - # - HashSize 32768 - - # - # Maximum number of conntracks, it should be double of: - # $ cat /proc/sys/net/netfilter/nf_conntrack_max - # since the daemon may keep some dead entries cached for possible - # retransmission during state synchronization. - # - HashLimit 131072 - - # - # Logfile: on (/var/log/conntrackd.log), off, or a filename - # Default: off - # - LogFile on - - # - # Syslog: on, off or a facility name (daemon (default) or local0..7) - # Default: off - # - #Syslog on - - # - # Lockfile - # - LockFile /var/lock/conntrack.lock - - # - # Unix socket configuration - # - UNIX { - Path /var/run/conntrackd.ctl - Backlog 20 - } - - # - # Netlink event socket buffer size. If you do not specify this clause, - # the default buffer size value in /proc/net/core/rmem_default is - # used. This default value is usually around 100 Kbytes which is - # fairly small for busy firewalls. This leads to event message dropping - # and high CPU consumption. This example configuration file sets the - # size to 2 MBytes to avoid this sort of problems. - # - NetlinkBufferSize 2097152 - - # - # The daemon doubles the size of the netlink event socket buffer size - # if it detects netlink event message dropping. This clause sets the - # maximum buffer size growth that can be reached. This example file - # sets the size to 8 MBytes. - # - NetlinkBufferSizeMaxGrowth 8388608 - - # - # If the daemon detects that Netlink is dropping state-change events, - # it automatically schedules a resynchronization against the Kernel - # after 30 seconds (default value). Resynchronizations are expensive - # in terms of CPU consumption since the daemon has to get the full - # kernel state-table and purge state-entries that do not exist anymore. - # Be careful of setting a very small value here. You have the following - # choices: On (enabled, use default 30 seconds value), Off (disabled) - # or Value (in seconds, to set a specific amount of time). If not - # specified, the daemon assumes that this option is enabled. - # - # NetlinkOverrunResync On - - # - # If you want reliable event reporting over Netlink, set on this - # option. If you set on this clause, it is a good idea to set off - # NetlinkOverrunResync. This option is off by default and you need - # a Linux kernel >= 2.6.31. - # - # NetlinkEventsReliable Off - - # - # By default, the daemon receives state updates following an - # event-driven model. You can modify this behaviour by switching to - # polling mode with the PollSecs clause. This clause tells conntrackd - # to dump the states in the kernel every N seconds. With regards to - # synchronization mode, the polling mode can only guarantee that - # long-lifetime states are recovered. The main advantage of this method - # is the reduction in the state replication at the cost of reducing the - # chances of recovering connections. - # - # PollSecs 15 - - # - # The daemon prioritizes the handling of state-change events coming - # from the core. With this clause, you can set the maximum number of - # state-change events (those coming from kernel-space) that the daemon - # will handle after which it will handle other events coming from the - # network or userspace. A low value improves interactivity (in terms of - # real-time behaviour) at the cost of extra CPU consumption. - # Default (if not set) is 100. - # - # EventIterationLimit 100 - - # - # Event filtering: This clause allows you to filter certain traffic, - # There are currently three filter-sets: Protocol, Address and - # State. The filter is attached to an action that can be: Accept or - # Ignore. Thus, you can define the event filtering policy of the - # filter-sets in positive or negative logic depending on your needs. - # You can select if conntrackd filters the event messages from - # user-space or kernel-space. The kernel-space event filtering - # saves some CPU cycles by avoiding the copy of the event message - # from kernel-space to user-space. The kernel-space event filtering - # is prefered, however, you require a Linux kernel >= 2.6.29 to - # filter from kernel-space. If you want to select kernel-space - # event filtering, use the keyword 'Kernelspace' instead of - # 'Userspace'. - # - Filter From Userspace { - # - # Accept only certain protocols: You may want to replicate - # the state of flows depending on their layer 4 protocol. - # - Protocol Accept { - TCP - SCTP - DCCP - # UDP - # ICMP # This requires a Linux kernel >= 2.6.31 - } - - # - # Ignore traffic for a certain set of IP's: Usually all the - # IP assigned to the firewall since local traffic must be - # ignored, only forwarded connections are worth to replicate. - # Note that these values depends on the local IPs that are - # assigned to the firewall. - # - Address Ignore { - IPv4_address 127.0.0.1 # loopback - IPv4_address [IGNORE_IP1] - IPv4_address [IGNORE_IP2] - IPv4_address [IGNORE_IP3] - #IPv4_address 192.168.0.100 # virtual IP 1 - #IPv4_address 192.168.1.100 # virtual IP 2 - #IPv4_address 192.168.0.1 - #IPv4_address 192.168.1.1 - #IPv4_address 192.168.100.100 # dedicated link ip - # - # You can also specify networks in format IP/cidr. - # IPv4_address 192.168.0.0/24 - # - # You can also specify an IPv6 address - # IPv6_address ::1 - } - - # - # Uncomment this line below if you want to filter by flow state. - # This option introduces a trade-off in the replication: it - # reduces CPU consumption at the cost of having lazy backup - # firewall replicas. The existing TCP states are: SYN_SENT, - # SYN_RECV, ESTABLISHED, FIN_WAIT, CLOSE_WAIT, LAST_ACK, - # TIME_WAIT, CLOSED, LISTEN. - # - # State Accept { - # ESTABLISHED CLOSED TIME_WAIT CLOSE_WAIT for TCP - # } - } -} diff --git a/systemvm/pom.xml b/systemvm/pom.xml index 0ae3e029ae9..e5d252313d9 100644 --- a/systemvm/pom.xml +++ b/systemvm/pom.xml @@ -88,7 +88,7 @@ - patches/debian/root/.ssh + debian/root/.ssh authorized_keys @@ -112,10 +112,7 @@ - - - - + diff --git a/systemvm/patches/test/TestCsAddress.py b/systemvm/test/TestCsAddress.py similarity index 100% rename from systemvm/patches/test/TestCsAddress.py rename to systemvm/test/TestCsAddress.py diff --git a/systemvm/patches/test/TestCsApp.py b/systemvm/test/TestCsApp.py similarity index 100% rename from systemvm/patches/test/TestCsApp.py rename to systemvm/test/TestCsApp.py diff --git a/systemvm/patches/test/TestCsCmdLine.py b/systemvm/test/TestCsCmdLine.py similarity index 100% rename from systemvm/patches/test/TestCsCmdLine.py rename to systemvm/test/TestCsCmdLine.py diff --git a/systemvm/patches/test/TestCsConfig.py b/systemvm/test/TestCsConfig.py similarity index 100% rename from systemvm/patches/test/TestCsConfig.py rename to systemvm/test/TestCsConfig.py diff --git a/systemvm/patches/test/TestCsDatabag.py b/systemvm/test/TestCsDatabag.py similarity index 100% rename from systemvm/patches/test/TestCsDatabag.py rename to systemvm/test/TestCsDatabag.py diff --git a/systemvm/patches/test/TestCsDhcp.py b/systemvm/test/TestCsDhcp.py similarity index 100% rename from systemvm/patches/test/TestCsDhcp.py rename to systemvm/test/TestCsDhcp.py diff --git a/systemvm/patches/test/TestCsFile.py b/systemvm/test/TestCsFile.py similarity index 100% rename from systemvm/patches/test/TestCsFile.py rename to systemvm/test/TestCsFile.py diff --git a/systemvm/patches/test/TestCsGuestNetwork.py b/systemvm/test/TestCsGuestNetwork.py similarity index 94% rename from systemvm/patches/test/TestCsGuestNetwork.py rename to systemvm/test/TestCsGuestNetwork.py index 08a6b372333..06e2d16cc4a 100644 --- a/systemvm/patches/test/TestCsGuestNetwork.py +++ b/systemvm/test/TestCsGuestNetwork.py @@ -17,6 +17,7 @@ import unittest from cs.CsGuestNetwork import CsGuestNetwork +from cs.CsConfig import CsConfig import merge @@ -30,7 +31,7 @@ class TestCsGuestNetwork(unittest.TestCase): self.assertTrue(csguestnetwork is not None) def test_get_dns(self): - csguestnetwork = CsGuestNetwork({}, {}) + csguestnetwork = CsGuestNetwork({}, CsConfig()) csguestnetwork.guest = True csguestnetwork.set_dns("1.1.1.1,2.2.2.2") csguestnetwork.set_router("3.3.3.3") diff --git a/systemvm/patches/test/TestCsHelper.py b/systemvm/test/TestCsHelper.py similarity index 100% rename from systemvm/patches/test/TestCsHelper.py rename to systemvm/test/TestCsHelper.py diff --git a/systemvm/patches/test/TestCsInterface.py b/systemvm/test/TestCsInterface.py similarity index 100% rename from systemvm/patches/test/TestCsInterface.py rename to systemvm/test/TestCsInterface.py diff --git a/systemvm/patches/test/TestCsNetfilter.py b/systemvm/test/TestCsNetfilter.py similarity index 100% rename from systemvm/patches/test/TestCsNetfilter.py rename to systemvm/test/TestCsNetfilter.py diff --git a/systemvm/patches/test/TestCsProcess.py b/systemvm/test/TestCsProcess.py similarity index 100% rename from systemvm/patches/test/TestCsProcess.py rename to systemvm/test/TestCsProcess.py diff --git a/systemvm/patches/test/TestCsRedundant.py b/systemvm/test/TestCsRedundant.py similarity index 100% rename from systemvm/patches/test/TestCsRedundant.py rename to systemvm/test/TestCsRedundant.py diff --git a/systemvm/patches/test/TestCsRoute.py b/systemvm/test/TestCsRoute.py similarity index 87% rename from systemvm/patches/test/TestCsRoute.py rename to systemvm/test/TestCsRoute.py index 6035258aa73..f9982fe1f8f 100644 --- a/systemvm/patches/test/TestCsRoute.py +++ b/systemvm/test/TestCsRoute.py @@ -31,11 +31,12 @@ class TestCsRoute(unittest.TestCase): def test_defaultroute_exists(self): csroute = CsRoute() - self.assertFalse(csroute.defaultroute_exists()) + self.assertTrue(csroute.defaultroute_exists()) def test_add_defaultroute(self): csroute = CsRoute() - self.assertTrue(csroute.add_defaultroute("192.168.1.1")) + # Default route would exist on any test environment with network + self.assertFalse(csroute.add_defaultroute("192.168.1.1")) def test_get_tablename(self): csroute = CsRoute() diff --git a/systemvm/patches/test/TestCsRule.py b/systemvm/test/TestCsRule.py similarity index 100% rename from systemvm/patches/test/TestCsRule.py rename to systemvm/test/TestCsRule.py diff --git a/systemvm/patches/test/runtests.sh b/systemvm/test/runtests.sh similarity index 67% rename from systemvm/patches/test/runtests.sh rename to systemvm/test/runtests.sh index e64f8674a08..04290e77608 100644 --- a/systemvm/patches/test/runtests.sh +++ b/systemvm/test/runtests.sh @@ -21,7 +21,24 @@ export PYTHONPATH="../debian/opt/cloud/bin/" export PYTHONDONTWRITEBYTECODE=False -pep8 --max-line-length=179 --exclude=monitorServices.py,baremetal-vr.py,passwd_server_ip.py `find ../../patches -name \*.py` +echo "Running pep8 to check systemvm/python code for errors" pep8 --max-line-length=179 *py +pep8 --max-line-length=179 --exclude=monitorServices.py,baremetal-vr.py,passwd_server_ip.py `find ../debian -name \*.py` +if [ $? -gt 0 ] +then + echo "Pylint failed, please check your code" + exit 1 +fi +echo "Running pylint to check systemvm/python code for errors" +pylint --disable=R,C,W *.py +pylint --disable=R,C,W `find ../debian -name \*.py` +if [ $? -gt 0 ] +then + echo "Pylint failed, please check your code" + exit 1 +fi + +echo "Running systemvm/python unit tests" nosetests . +exit $? diff --git a/test/integration/smoke/test_deploy_virtio_scsi_vm.py b/test/integration/smoke/test_deploy_virtio_scsi_vm.py index fbff31c3a9a..52ef25470fd 100644 --- a/test/integration/smoke/test_deploy_virtio_scsi_vm.py +++ b/test/integration/smoke/test_deploy_virtio_scsi_vm.py @@ -130,7 +130,6 @@ class TestDeployVirtioSCSIVM(cloudstackTestCase): cls.services["service_offerings"]["small"] ) - cls.sparse_disk_offering = DiskOffering.create( cls.apiclient, cls.services["sparse_disk_offering"] diff --git a/test/integration/smoke/test_deploy_vm_iso.py b/test/integration/smoke/test_deploy_vm_iso.py index 76ba2b7ec5f..5298c48ee19 100644 --- a/test/integration/smoke/test_deploy_vm_iso.py +++ b/test/integration/smoke/test_deploy_vm_iso.py @@ -130,7 +130,7 @@ class TestDeployVMFromISO(cloudstackTestCase): ) try: # Download the ISO - self.iso.download(self.apiclient) + self.iso.download(self.apiclient, retries=150) except Exception as e: raise Exception("Exception while downloading ISO %s: %s" diff --git a/test/integration/smoke/test_hosts.py b/test/integration/smoke/test_hosts.py deleted file mode 100644 index 952f1600949..00000000000 --- a/test/integration/smoke/test_hosts.py +++ /dev/null @@ -1,170 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -""" BVT tests for Hosts and Clusters -""" -#Import Local Modules -import marvin -from marvin.cloudstackTestCase import * -from marvin.cloudstackAPI import * -from marvin.lib.utils import * -from marvin.lib.base import * -from marvin.lib.common import * -from marvin.lib.utils import (random_gen) -from nose.plugins.attrib import attr - -#Import System modules -import time - -_multiprocess_shared_ = True - -class TestHosts(cloudstackTestCase): - - def setUp(self): - self.apiclient = self.testClient.getApiClient() - self.hypervisor = self.testClient.getHypervisorInfo() - self.dbclient = self.testClient.getDbConnection() - self.services = self.testClient.getParsedTestDataConfig() - self.zone = get_zone(self.apiclient, self.testClient.getZoneForTests()) - self.pod = get_pod(self.apiclient, self.zone.id) - self.cleanup = [] - - return - - def tearDown(self): - try: - #Clean up, terminate the created templates - cleanup_resources(self.apiclient, self.cleanup) - - except Exception as e: - raise Exception("Warning: Exception during cleanup : %s" % e) - return - - #@attr(tags=["selfservice"]) - def test_01_clusters(self): - """Test Add clusters & hosts - simulator - - - # Validate the following: - # 1. Verify hypervisortype returned by API is Simulator/Xen/KVM/VWare - # 2. Verify that the cluster is in 'Enabled' allocation state - # 3. Verify that the host is added successfully and in Up state - # with listHosts API response - - #Create clusters with Hypervisor type Simulator/XEN/KVM/VWare - """ - for k, v in self.services["clusters"].items(): - v["clustername"] = v["clustername"] + "-" + random_gen() - cluster = Cluster.create( - self.apiclient, - v, - zoneid=self.zone.id, - podid=self.pod.id, - hypervisor=v["hypervisor"].lower() - ) - self.debug( - "Created Cluster for hypervisor type %s & ID: %s" %( - v["hypervisor"], - cluster.id - )) - self.assertEqual( - cluster.hypervisortype.lower(), - v["hypervisor"].lower(), - "Check hypervisor type is " + v["hypervisor"] + " or not" - ) - self.assertEqual( - cluster.allocationstate, - 'Enabled', - "Check whether allocation state of cluster is enabled" - ) - - #If host is externally managed host is already added with cluster - response = list_hosts( - self.apiclient, - clusterid=cluster.id - ) - - if not response: - hypervisor_type = str(cluster.hypervisortype.lower()) - host = Host.create( - self.apiclient, - cluster, - self.services["hosts"][hypervisor_type], - zoneid=self.zone.id, - podid=self.pod.id, - hypervisor=v["hypervisor"].lower() - ) - if host == FAILED: - self.fail("Host Creation Failed") - self.debug( - "Created host (ID: %s) in cluster ID %s" %( - host.id, - cluster.id - )) - #Cleanup Host & Cluster - self.cleanup.append(host) - self.cleanup.append(cluster) - - list_hosts_response = list_hosts( - self.apiclient, - clusterid=cluster.id - ) - self.assertEqual( - isinstance(list_hosts_response, list), - True, - "Check list response returns a valid list" - ) - self.assertNotEqual( - len(list_hosts_response), - 0, - "Check list Hosts response" - ) - - host_response = list_hosts_response[0] - #Check if host is Up and running - self.assertEqual( - host_response.state, - 'Up', - "Check if state of host is Up or not" - ) - #Verify List Cluster Response has newly added cluster - list_cluster_response = list_clusters( - self.apiclient, - id=cluster.id - ) - self.assertEqual( - isinstance(list_cluster_response, list), - True, - "Check list response returns a valid list" - ) - self.assertNotEqual( - len(list_cluster_response), - 0, - "Check list Hosts response" - ) - - cluster_response = list_cluster_response[0] - self.assertEqual( - cluster_response.id, - cluster.id, - "Check cluster ID with list clusters response" - ) - self.assertEqual( - cluster_response.hypervisortype.lower(), - cluster.hypervisortype.lower(), - "Check hypervisor type with is " + v["hypervisor"] + " or not" - ) - return diff --git a/test/integration/smoke/test_privategw_acl.py b/test/integration/smoke/test_privategw_acl.py index bba33eb947b..b86b96ea031 100644 --- a/test/integration/smoke/test_privategw_acl.py +++ b/test/integration/smoke/test_privategw_acl.py @@ -28,6 +28,8 @@ from marvin.codes import PASS import time import logging +import random + class Services: """Test VPC network services - Port Forwarding Rules Test Data Class. @@ -243,14 +245,19 @@ class TestPrivateGwACL(cloudstackTestCase): qresultset = self.dbclient.execute( "select vnet from op_dc_vnet_alloc where physical_network_id=\ - (select id from physical_network where uuid='%s' ) and taken is NULL and reservation_id is NULL and account_id is NULL ORDER BY id DESC;" % physical_network.id - ) + (select id from physical_network where uuid='%s');" % physical_network.id) self.assertEqual(validateList(qresultset)[0], PASS, "Invalid sql query response" ) - vlans = qresultset - vlan_1 = int(vlans[0][0]) + + # Find all the vlans that are for dynamic vlan allocation + dc_vlans = sorted(map(lambda x: x[0], qresultset)) + + # Use VLAN id that is not in physical network vlan range for dynamic vlan allocation + vlan_1 = int(physical_network.vlan.split('-')[-1]) + 1 + if vlan_1 in dc_vlans: + vlan_1 = dc_vlans[-1] + random.randint(1, 5) acl = self.createACL(vpc) self.createACLItem(acl.id) @@ -328,14 +335,19 @@ class TestPrivateGwACL(cloudstackTestCase): qresultset = self.dbclient.execute( "select vnet from op_dc_vnet_alloc where physical_network_id=\ - (select id from physical_network where uuid='%s' ) and taken is NULL and reservation_id is NULL and account_id is NULL ORDER BY id DESC;" % physical_network.id - ) + (select id from physical_network where uuid='%s');" % physical_network.id) self.assertEqual(validateList(qresultset)[0], PASS, "Invalid sql query response" - ) - vlans = qresultset - vlan_1 = int(vlans[0][0]) + ) + + # Find all the vlans that are for dynamic vlan allocation + dc_vlans = sorted(map(lambda x: x[0], qresultset)) + + # Use VLAN id that is not in physical network vlan range for dynamic vlan allocation + vlan_1 = int(physical_network.vlan.split('-')[-1]) + 1 + if vlan_1 in dc_vlans: + vlan_1 = dc_vlans[-1] + random.randint(1, 5) acl1 = self.createACL(vpc_1) self.createACLItem(acl1.id, cidr = "0.0.0.0/0") @@ -376,13 +388,25 @@ class TestPrivateGwACL(cloudstackTestCase): physical_network = self.get_guest_traffic_physical_network(self.apiclient, self.zone.id) if not physical_network: self.fail("No Physical Networks found!") + qresultset = self.dbclient.execute( "select vnet from op_dc_vnet_alloc where physical_network_id=\ - (select id from physical_network where uuid='%s' ) and taken is NULL and reservation_id is NULL and account_id is NULL ORDER BY id DESC;" % physical_network.id - ) - vlans = qresultset - vlan_1 = int(vlans[0][0]) + (select id from physical_network where uuid='%s');" % physical_network.id) + self.assertEqual(validateList(qresultset)[0], + PASS, + "Invalid sql query response" + ) + # Find all the vlans that are for dynamic vlan allocation + dc_vlans = sorted(map(lambda x: x[0], qresultset)) + + # Use VLAN id that is not in physical network vlan range for dynamic vlan allocation + vlan_1 = int(physical_network.vlan.split('-')[-1]) + 1 + if vlan_1 in dc_vlans: + vlan_1 = dc_vlans[-1] + random.randint(1, 5) + + acl1 = self.createACL(vpc_1) + self.createACLItem(acl1.id, cidr = "0.0.0.0/0") net_offering_no_lb = "network_offering_no_lb" network_1 = self.createNetwork(vpc_1, gateway = '10.0.0.1') @@ -706,18 +730,17 @@ class TestPrivateGwACL(cloudstackTestCase): self.logger.debug("Sleeping for %s seconds in order to get the firewall applied..." % sleep_time) time.sleep(sleep_time) - sleep_time += sleep_time self.logger.debug("Ping to VM inside another Network Tier") result = str(ssh.execute(ssh_command)) - self.logger.debug("SSH result: %s; COUNT is ==> %s" % (result, result.count("3 received"))) + self.logger.debug("SSH result: %s; COUNT is ==> %s" % (result, result.count("0% packet loss"))) except Exception as e: self.fail("SSH Access failed for %s: %s" % \ (virtual_machine, e) ) - succeeded_pings += result.count("3 received") + succeeded_pings += result.count("0% packet loss") self.assertTrue(succeeded_pings >= minimum_vms_to_pass, diff --git a/test/integration/smoke/test_vpc_redundant.py b/test/integration/smoke/test_vpc_redundant.py index 2c7f4aeac61..372d8476710 100644 --- a/test/integration/smoke/test_vpc_redundant.py +++ b/test/integration/smoke/test_vpc_redundant.py @@ -275,7 +275,7 @@ class TestVPCRedundancy(cloudstackTestCase): cls.logger.setLevel(logging.DEBUG) cls.logger.addHandler(cls.stream_handler) - return + cls.advert_int = int(Configurations.list(cls.api_client, name="router.redundant.vrrp.interval")[0].value) @classmethod def tearDownClass(cls): @@ -283,7 +283,6 @@ class TestVPCRedundancy(cloudstackTestCase): cleanup_resources(cls.api_client, cls._cleanup) except Exception as e: raise Exception("Warning: Exception during cleanup : %s" % e) - return def setUp(self): self.routers = [] @@ -344,10 +343,16 @@ class TestVPCRedundancy(cloudstackTestCase): len(self.routers), count, "Check that %s routers were indeed created" % count) + def wait_for_vrrp(self): + # Wait until 3*advert_int+skew time to get one of the routers as MASTER + time.sleep(3 * self.advert_int + 5) + def check_routers_state(self,count=2, status_to_check="MASTER", expected_count=1, showall=False): vals = ["MASTER", "BACKUP", "UNKNOWN"] cnts = [0, 0, 0] + self.wait_for_vrrp() + result = "UNKNOWN" self.query_routers(count, showall) for router in self.routers: @@ -404,6 +409,7 @@ class TestVPCRedundancy(cloudstackTestCase): self.logger.debug('Stopping router %s' % router.id) cmd = stopRouter.stopRouterCmd() cmd.id = router.id + cmd.forced = True self.apiclient.stopRouter(cmd) def reboot_router(self, router): @@ -657,7 +663,7 @@ class TestVPCRedundancy(cloudstackTestCase): @attr(tags=["advanced", "intervlan"], required_hardware="true") def test_05_rvpc_multi_tiers(self): """ Create a redundant VPC with 1 Tier, 1 VM, 1 ACL, 1 PF and test Network GC Nics""" - self.logger.debug("Starting test_04_rvpc_network_garbage_collector_nics") + self.logger.debug("Starting test_05_rvpc_multi_tiers") self.query_routers() network = self.create_network(self.services["network_offering"], "10.1.1.1", nr_vms=1, mark_net_cleanup=False) @@ -732,7 +738,7 @@ class TestVPCRedundancy(cloudstackTestCase): ssh_command = "ping -c 3 8.8.8.8" # Should be able to SSH VM - result = 'failed' + packet_loss = 100 try: vm = vmObj.get_vm() public_ip = vmObj.get_ip() @@ -741,19 +747,22 @@ class TestVPCRedundancy(cloudstackTestCase): ssh = vm.get_ssh_client(ipaddress=public_ip.ipaddress.ipaddress) self.logger.debug("Ping to google.com from VM") - result = str(ssh.execute(ssh_command)) + result = ssh.execute(ssh_command) - self.logger.debug("SSH result: %s; COUNT is ==> %s" % (result, result.count(" 0% packet loss"))) + for line in result: + if "packet loss" in line: + packet_loss = int(line.split("% packet loss")[0].split(" ")[-1]) + break + + self.logger.debug("SSH result: %s; packet loss is ==> %s" % (result, packet_loss)) except Exception as e: self.fail("SSH Access failed for %s: %s" % \ (vmObj.get_ip(), e) ) - - self.assertEqual( - result.count(" 0% packet loss"), - 1, - "Ping to outside world from VM should be successful" - ) + + # Most pings should be successful + self.assertTrue(packet_loss < 50, + "Ping to outside world from VM should be successful") class networkO(object): diff --git a/tools/appliance/definitions/systemvmtemplate/apt_upgrade.sh b/tools/appliance/definitions/systemvmtemplate/apt_upgrade.sh index eb9b7901d6d..6f37e343fef 100644 --- a/tools/appliance/definitions/systemvmtemplate/apt_upgrade.sh +++ b/tools/appliance/definitions/systemvmtemplate/apt_upgrade.sh @@ -19,6 +19,15 @@ set -e set -x +function remove_reserved_blocks() { + for partition in $(blkid -o list | grep ext | awk '{print $1}') + do + tune2fs -m0 $partition + done + fdisk -l + df -h +} + function add_backports() { sed -i '/cdrom/d' /etc/apt/sources.list sed -i '/deb-src/d' /etc/apt/sources.list @@ -33,6 +42,7 @@ function apt_upgrade() { # Setup sudo echo 'cloud ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/cloud + remove_reserved_blocks add_backports rm -fv /root/*.iso @@ -40,8 +50,6 @@ function apt_upgrade() { apt-get autoclean apt-get -q -y update apt-get -q -y upgrade - - df -h } return 2>/dev/null || apt_upgrade diff --git a/tools/appliance/definitions/systemvmtemplate/cleanup.sh b/tools/appliance/definitions/systemvmtemplate/cleanup.sh index 444cc948f17..00f068d2fa7 100644 --- a/tools/appliance/definitions/systemvmtemplate/cleanup.sh +++ b/tools/appliance/definitions/systemvmtemplate/cleanup.sh @@ -20,8 +20,11 @@ set -e set -x function cleanup_apt() { - apt-get -y remove dictionaries-common busybox isc-dhcp-client isc-dhcp-common - apt-get -y autoremove + apt-get -y remove --purge dictionaries-common busybox isc-dhcp-client isc-dhcp-common \ + task-english task-ssh-server tasksel tasksel-data laptop-detect nano wamerican \ + debconf-i18n sharutils gnupg gnupg-agent keyboard-configuration + + apt-get -y autoremove --purge apt-get autoclean apt-get clean } @@ -40,13 +43,28 @@ function cleanup_dev() { } function cleanup_misc() { + # Scripts rm -fr /home/cloud/cloud_scripts* + rm -f /usr/share/cloud/cloud-scripts.tar rm -f /root/.rnd rm -f /var/www/html/index.html + # Logs rm -f /var/log/*.log rm -f /var/log/apache2/* rm -f /var/log/messages rm -f /var/log/syslog + rm -f /var/log/messages + rm -fr /var/log/apt + rm -fr /var/log/installer + # Docs and data files + rm -fr /var/lib/apt/* + rm -fr /var/cache/apt/* + rm -fr /usr/lib/gnupg* + rm -fr /usr/share/doc + rm -fr /usr/share/man + rm -fr /usr/share/info + rm -fr /usr/share/lintian + find /usr/share/locale -type f | grep -v en | xargs rm -fr } function cleanup() { diff --git a/tools/appliance/definitions/systemvmtemplate/configure_login.sh b/tools/appliance/definitions/systemvmtemplate/configure_login.sh index d5f80b105fc..1d9de06d45b 100644 --- a/tools/appliance/definitions/systemvmtemplate/configure_login.sh +++ b/tools/appliance/definitions/systemvmtemplate/configure_login.sh @@ -27,54 +27,22 @@ function configure_user() { usermod -a -G admin cloud mkdir -p /home/cloud/.ssh chmod 700 /home/cloud/.ssh - echo "cloud:`openssl rand -base64 32`" | chpasswd echo "root:password" | chpasswd } -function configure_sudoers() { - cat >/etc/sudoers < /etc/sudoers.d/cloud -} - -# sshd_config is overwritten from cloud_scripts -#function configure_sshd() { -# grep "UseDNS no" /etc/ssh/sshd_config && \ -# grep "PasswordAuthentication no" /etc/ssh/sshd_config && \ -# return -# # Tweak sshd to prevent DNS resolution (speed up logins) -# echo 'UseDNS no' >> /etc/ssh/sshd_config -# -# # Require ssh keys for login -# sed -i -e 's/^.*PasswordAuthentication .*$/PasswordAuthentication no/g' /etc/ssh/sshd_config -#} - function configure_inittab() { # Fix inittab cat >> /etc/inittab << EOF +0:2345:respawn:/sbin/getty -L 115200 ttyS0 vt102 vc:2345:respawn:/sbin/getty 38400 hvc0 EOF } function configure_login() { - # configure_sshd configure_inittab add_admin_group - configure_sudoers configure_user - rm -fv /home/cloud/configure_login.sh - sync - halt -p } return 2>/dev/null || configure_login diff --git a/tools/appliance/definitions/systemvmtemplate/configure_systemvm_services.sh b/tools/appliance/definitions/systemvmtemplate/configure_systemvm_services.sh index 791f2ead984..84cb5909c3e 100644 --- a/tools/appliance/definitions/systemvmtemplate/configure_systemvm_services.sh +++ b/tools/appliance/definitions/systemvmtemplate/configure_systemvm_services.sh @@ -34,70 +34,15 @@ function configure_apache2() { function install_cloud_scripts() { # ./cloud_scripts/ has been put there by ../../cloud_scripts_shar_archive.sh rsync -av ./cloud_scripts/ / - chmod +x /opt/cloud/bin/* \ + chmod +x /opt/cloud/bin/* /opt/cloud/bin/setup/* \ /root/{clearUsageRules.sh,reconfigLB.sh,monitorServices.py} \ - /etc/init.d/{cloud-early-config,cloud-passwd-srvr} \ /etc/profile.d/cloud.sh - cat > /etc/systemd/system/cloud-early-config.service << EOF -[Unit] -Description=cloud-early-config: configures systemvm using cmdline -DefaultDependencies=no - -Before=network-pre.target -Wants=network-pre.target - -Requires=local-fs.target -After=local-fs.target - -[Install] -WantedBy=multi-user.target - -[Service] -Type=oneshot -ExecStart=/etc/init.d/cloud-early-config start -ExecStop=/etc/init.d/cloud-early-config stop -RemainAfterExit=true -TimeoutStartSec=5min - -EOF - - cat > /etc/systemd/system/cloud.service << EOF -[Unit] -Description=cloud: startup cloud service -After=cloud-early-config.service network.target local-fs.target - -[Install] -WantedBy=multi-user.target - -[Service] -Type=simple -WorkingDirectory=/usr/local/cloud/systemvm -ExecStart=/usr/local/cloud/systemvm/_run.sh -Restart=always -RestartSec=5 -EOF - - cat > /etc/systemd/system/cloud-passwd-srvr.service << EOF -[Unit] -Description=cloud-passwd-srvr: cloud password server -After=network.target local-fs.target - -[Install] -WantedBy=multi-user.target - -[Service] -Type=forking -ExecStart=/etc/init.d/cloud-passwd-srvr start -ExecStop=/etc/init.d/cloud-passwd-srvr stop -Restart=always -RestartSec=5 -EOF + chmod -x /etc/systemd/system/* systemctl daemon-reload systemctl enable cloud-early-config - systemctl disable cloud - systemctl disable cloud-passwd-srvr + systemctl enable cloud-postinit } function do_signature() { diff --git a/tools/appliance/definitions/systemvmtemplate/definition.rb b/tools/appliance/definitions/systemvmtemplate/definition.rb index 33c8e8bd02a..03be56845b6 100644 --- a/tools/appliance/definitions/systemvmtemplate/definition.rb +++ b/tools/appliance/definitions/systemvmtemplate/definition.rb @@ -81,11 +81,11 @@ config = { 'configure_systemvm_services.sh', 'authorized_keys.sh', 'configure_persistent_config.sh', + # setup login stuff + 'configure_login.sh', # cleanup & space-saving 'cleanup.sh', - 'zerodisk.sh', - # setup login stuff - 'configure_login.sh' + 'finalize.sh' ], :postinstall_timeout => '10000' } diff --git a/tools/appliance/definitions/systemvmtemplate/zerodisk.sh b/tools/appliance/definitions/systemvmtemplate/finalize.sh similarity index 68% rename from tools/appliance/definitions/systemvmtemplate/zerodisk.sh rename to tools/appliance/definitions/systemvmtemplate/finalize.sh index d002316d658..1a25e399e77 100644 --- a/tools/appliance/definitions/systemvmtemplate/zerodisk.sh +++ b/tools/appliance/definitions/systemvmtemplate/finalize.sh @@ -19,6 +19,26 @@ set -e set -x +function configure_misc() { + rm -fv /home/cloud/*.sh + echo "cloud:`openssl rand -base64 32`" | chpasswd +} + +function configure_sudoers() { + cat >/etc/sudoers < /etc/sudoers.d/cloud +} + # clean up stuff copied in by veewee function cleanup_veewee() { # this has to be here since it is the last file to run (and we remove ourselves) @@ -31,19 +51,22 @@ function cleanup_veewee() { # Zero out the free space to save space in the final image: function zero_disk() { - cleanup_veewee - for path in / /boot /var/log /tmp do dd if=/dev/zero of=${path}/zero bs=1M || true sync rm -f ${path}/zero done - - for partition in $(blkid -o list | grep ext | awk '{print $1}') - do - tune2fs -m0 $partition - done + df -h } -return 2>/dev/null || zero_disk +function finalize() { + configure_misc + configure_sudoers + cleanup_veewee + sync + zero_disk + halt -p +} + +return 2>/dev/null || finalize diff --git a/tools/appliance/definitions/systemvmtemplate/install_systemvm_packages.sh b/tools/appliance/definitions/systemvmtemplate/install_systemvm_packages.sh index 2cceb989a25..40ff6df4d68 100644 --- a/tools/appliance/definitions/systemvmtemplate/install_systemvm_packages.sh +++ b/tools/appliance/definitions/systemvmtemplate/install_systemvm_packages.sh @@ -53,7 +53,7 @@ function install_packages() { fi ${apt_get} install grub-legacy \ - rsyslog logrotate cron net-tools ifupdown vim tmux netbase iptables \ + rsyslog logrotate cron net-tools ifupdown vim netbase iptables \ openssh-server e2fsprogs tcpdump socat wget \ python bzip2 sed gawk diffutils grep gzip less tar telnet ftp rsync traceroute psmisc lsof procps \ inetutils-ping iputils-arping httping curl \ @@ -67,7 +67,6 @@ function install_packages() { xenstore-utils libxenstore3.0 \ ipvsadm conntrackd libnetfilter-conntrack3 \ keepalived irqbalance \ - libnl-3-200 libnl-genl-3-200 \ ipcalc \ openjdk-8-jre-headless \ ipset \ @@ -80,16 +79,14 @@ function install_packages() { strongswan libcharon-extra-plugins libstrongswan-extra-plugins \ virt-what open-vm-tools qemu-guest-agent hyperv-daemons + apt-get -y autoremove --purge + apt-get autoclean + apt-get clean + # Install xenserver guest utilities as debian repos don't have it wget https://mirrors.kernel.org/ubuntu/pool/universe/x/xe-guest-utilities/xe-guest-utilities_7.4.0-0ubuntu1_amd64.deb dpkg -i xe-guest-utilities_7.4.0-0ubuntu1_amd64.deb rm -f xe-guest-utilities_7.4.0-0ubuntu1_amd64.deb - - apt-get autoclean - apt-get clean - - apt-get update - apt-get -y upgrade } return 2>/dev/null || install_packages diff --git a/tools/appliance/definitions/systemvmtemplate/preseed.cfg b/tools/appliance/definitions/systemvmtemplate/preseed.cfg index 99bd6eb04ab..5a8e2d9fba9 100644 --- a/tools/appliance/definitions/systemvmtemplate/preseed.cfg +++ b/tools/appliance/definitions/systemvmtemplate/preseed.cfg @@ -57,7 +57,7 @@ d-i partman-auto/method string regular d-i partman-auto/choose_recipe select atomic d-i partman-auto/expert_recipe string \ boot-root :: \ - 80 50 100 ext2 \ + 50 50 50 ext2 \ $primary{ } $bootable{ } \ method{ format } format{ } \ use_filesystem{ } filesystem{ ext2 } \ @@ -68,17 +68,12 @@ d-i partman-auto/expert_recipe string \ use_filesystem{ } filesystem{ ext4 } \ mountpoint{ / } \ . \ - 300 50 300 ext4 \ + 350 50 400 ext4 \ method{ format } format{ } \ use_filesystem{ } filesystem{ ext4 } \ mountpoint{ /var/log } \ . \ - 100 90 100 ext4 \ - method{ format } format{ } \ - use_filesystem{ } filesystem{ ext4 } \ - mountpoint{ /tmp } \ - . \ - 100 100 200 linux-swap \ + 100 100 100 linux-swap \ method{ swap } format{ } \ . diff --git a/tools/appliance/shar_cloud_scripts.sh b/tools/appliance/shar_cloud_scripts.sh index 5504d16e3ae..30ae2cb6f40 100755 --- a/tools/appliance/shar_cloud_scripts.sh +++ b/tools/appliance/shar_cloud_scripts.sh @@ -37,14 +37,11 @@ TEMP_DIR=`mktemp -d ${TMPDIR}/shar_cloud.XXXXXXXX` cd ${TEMP_DIR} mkdir cloud_scripts mkdir -p cloud_scripts/opt/cloudstack -cp -r ${CLOUDSTACK_DIR}/systemvm/patches/debian/config/* cloud_scripts/ -cp -r ${CLOUDSTACK_DIR}/systemvm/patches/debian/vpn/* cloud_scripts/ +cp -r ${CLOUDSTACK_DIR}/systemvm/debian/* cloud_scripts/ mkdir -p cloud_scripts/usr/share/cloud -cd ${CLOUDSTACK_DIR}/systemvm/patches/debian/config +cd ${CLOUDSTACK_DIR}/systemvm/debian tar -cf ${TEMP_DIR}/cloud_scripts/usr/share/cloud/cloud-scripts.tar * -cd ${CLOUDSTACK_DIR}/systemvm/patches/debian/vpn -tar -rf ${TEMP_DIR}/cloud_scripts/usr/share/cloud/cloud-scripts.tar * cd ${TEMP_DIR} shar `find . -print` > ${CURR_DIR}/cloud_scripts_shar_archive.sh diff --git a/tools/marvin/marvin/config/test_data.py b/tools/marvin/marvin/config/test_data.py index e99c8ba0f38..bca541f6237 100644 --- a/tools/marvin/marvin/config/test_data.py +++ b/tools/marvin/marvin/config/test_data.py @@ -1780,24 +1780,24 @@ test_data = { "VHD": { "templatename": "XenUploadtemplate", "displaytext": "XenUploadtemplate", - "url": "http://people.apache.org/~sanjeev/centos56-x86_64.vhd.bz2", + "url": "http://dl.openvm.eu/cloudstack/macchinina/x86_64/macchinina-xen.vhd.bz2", "hypervisor":"XenServer", - "checksum": "09b08b6abb1b903fca7711d3ac8d6598", + "checksum": "54ebc933e6e07ae58c0dc97dfd37c824", "ostypeid":"74affaea-c658-11e4-ad38-a6d1374244b4" }, "OVA": { "templatename": "VMwareUploadtemplate", "displaytext": "VMwareUploadtemplate", - "url": "http://people.apache.org/~sanjeev/CentOS5.3-x86_64.ova", - "checksum": "02de0576dd3a61ab59c03fd795fc86ac", + "url": "http://dl.openvm.eu/cloudstack/macchinina/x86_64/macchinina-vmware.ova", + "checksum": "d6d97389b129c7d898710195510bf4fb", "hypervisor":"VMware", "ostypeid":"74affaea-c658-11e4-ad38-a6d1374244b4" }, "QCOW2": { "templatename": "KVMUploadtemplate", "displaytext": "VMwareUploadtemplate", - "url": "http://people.apache.org/~sanjeev/eec2209b-9875-3c8d-92be-c001bd8a0faf.qcow2.bz2", - "checksum": "da997b697feaa2f1f6e0d4785b0cece2", + "url": "http://dl.openvm.eu/cloudstack/macchinina/x86_64/macchinina-kvm.qcow2.bz2", + "checksum": "ada77653dcf1e59495a9e1ac670ad95f", "hypervisor":"KVM", "ostypeid":"2e02e376-cdf3-11e4-beb3-8aa6272b57ef" }, @@ -1883,7 +1883,7 @@ test_data = { "name": "testISO", "bootable": True, "ispublic": False, - "url": "http://people.apache.org/~sanjeev/CentOS-6.3-x86_64-bin-DVD1.iso", + "url": "http://dl.openvm.eu/cloudstack/iso/TinyCore-8.0.iso", "ostype": 'CentOS 6.3 (64-bit)', "mode": 'HTTP_DOWNLOAD' }, diff --git a/tools/travis/before_install.sh b/tools/travis/before_install.sh index 409bf8a7ffd..c31213ee052 100755 --- a/tools/travis/before_install.sh +++ b/tools/travis/before_install.sh @@ -100,7 +100,7 @@ pip install --user --upgrade pip for ((i=0;i<$RETRY_COUNT;i++)) do - pip install --user --upgrade lxml paramiko nose texttable ipmisim pyopenssl > /tmp/piplog + pip install --user --upgrade lxml paramiko nose texttable ipmisim pyopenssl mock flask netaddr pylint pep8 > /tmp/piplog if [[ $? -eq 0 ]]; then echo -e "\npython packages installed successfully" break; diff --git a/tools/travis/install.sh b/tools/travis/install.sh index d759528208f..6744979b399 100755 --- a/tools/travis/install.sh +++ b/tools/travis/install.sh @@ -25,9 +25,13 @@ export MAVEN_OPTS="-Xmx4096m -XX:MaxPermSize=800m -Djava.security.egd=file:/dev/ set -e if [ $TEST_SEQUENCE_NUMBER -eq 1 ]; then + # Pylint/pep8 systemvm python codebase + cd systemvm/test && bash -x runtests.sh && cd ../.. + # Build noredist git clone https://github.com/rhtyd/cloudstack-nonoss.git nonoss cd nonoss && bash -x install-non-oss.sh && cd .. git clean -fdx . + # Perform rat checks mvn -P developer,systemvm -Dsimulator -Dnoredist --projects='org.apache.cloudstack:cloudstack' org.apache.rat:apache-rat-plugin:0.12:check mvn -q -B -P developer,systemvm -Dsimulator -Dnoredist clean install else diff --git a/tools/whisker/LICENSE b/tools/whisker/LICENSE index d14a974e2c1..e107c4fc8d5 100644 --- a/tools/whisker/LICENSE +++ b/tools/whisker/LICENSE @@ -2758,7 +2758,7 @@ Within the deps/awsapi-lib directory from VMware, Inc http://www.vmware.com/ rabbitmq-client.jar from http://www.rabbitmq.com/java-client.html -Within the patches/systemvm/debian/config/etc directory +Within the systemvm/debian/etc directory placed in the public domain by Adiscon GmbH http://www.adiscon.com/ rsyslog.conf @@ -2766,14 +2766,14 @@ Within the patches/systemvm/debian/config/etc directory dnsmasq.conf vpcdnsmasq.conf -Within the patches/systemvm/debian/config/etc/apache2 directory +Within the systemvm/debian/config/etc/apache2 directory licensed under the Apache License, Version 2 http://www.apache.org/licenses/LICENSE-2.0.txt (as above) Copyright (c) 2012 The Apache Software Foundation from The Apache Software Foundation http://www.apache.org/ httpd.conf vhost.template -Within the patches/systemvm/debian/config/etc/ssh/ directory +Within the systemvm/debian/etc/ssh/ directory licensed under the BSD (2-clause) http://www.opensource.org/licenses/BSD-2-Clause (as follows) @@ -2804,7 +2804,7 @@ Within the patches/systemvm/debian/config/etc/ssh/ directory from OpenSSH Project http://www.openssh.org/ sshd_config -Within the patches/systemvm/debian/config/root/redundant_router directory +Within the systemvm/debian/root/redundant_router directory placed in the public domain by The netfilter.org project http://www.netfilter.org/ conntrackd.conf.templ diff --git a/tools/whisker/descriptor-for-packaging.xml b/tools/whisker/descriptor-for-packaging.xml index bc78b56e876..3a7624d28f5 100644 --- a/tools/whisker/descriptor-for-packaging.xml +++ b/tools/whisker/descriptor-for-packaging.xml @@ -2452,7 +2452,7 @@ This product includes software developed at The Apache Software Foundation (http://www.apache.org/). - + Copyright (c) 2012 The Apache Software Foundation @@ -2463,7 +2463,7 @@ Copyright (c) 2012 The Apache Software Foundation - + @@ -2474,14 +2474,14 @@ Copyright (c) 2012 The Apache Software Foundation - + - + diff --git a/tools/whisker/descriptor.xml b/tools/whisker/descriptor.xml index 2bc2abaf9ac..3fe1047686e 100644 --- a/tools/whisker/descriptor.xml +++ b/tools/whisker/descriptor.xml @@ -2436,7 +2436,7 @@ This product includes software developed at The Apache Software Foundation (http://www.apache.org/). - + Copyright (c) 2012 The Apache Software Foundation @@ -2447,7 +2447,7 @@ Copyright (c) 2012 The Apache Software Foundation - + @@ -2458,14 +2458,14 @@ Copyright (c) 2012 The Apache Software Foundation - + - + diff --git a/utils/src/main/java/com/cloud/utils/nio/Link.java b/utils/src/main/java/com/cloud/utils/nio/Link.java index 35211c8a458..25f6662c522 100644 --- a/utils/src/main/java/com/cloud/utils/nio/Link.java +++ b/utils/src/main/java/com/cloud/utils/nio/Link.java @@ -505,7 +505,7 @@ public class Link { s_logger.error(String.format("SSL error caught during unwrap data: %s, for local address=%s, remote address=%s. The client may have invalid ca-certificates.", sslException.getMessage(), socketChannel.getLocalAddress(), socketChannel.getRemoteAddress())); sslEngine.closeOutbound(); - return new HandshakeHolder(peerAppData, peerNetData, true); + return new HandshakeHolder(peerAppData, peerNetData, false); } if (result == null) { return new HandshakeHolder(peerAppData, peerNetData, false);