From 0f2012e842c58b6d2394466dd456ea3eb9c8297e Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 09:54:13 +0200 Subject: [PATCH 01/60] CLOUDSTACK-7143: upgrade systemvm to latest debian stable, 7.6.0. --- .../appliance/definitions/systemvm64template/definition.rb | 6 +++--- tools/appliance/definitions/systemvmtemplate/definition.rb | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tools/appliance/definitions/systemvm64template/definition.rb b/tools/appliance/definitions/systemvm64template/definition.rb index 4186742d681..2aaac04fdd2 100644 --- a/tools/appliance/definitions/systemvm64template/definition.rb +++ b/tools/appliance/definitions/systemvm64template/definition.rb @@ -10,9 +10,9 @@ Veewee::Definition.declare({ :memory_size=> '256', :disk_size => '2500', :disk_format => 'VDI', :hostiocache => 'off', :os_type_id => 'Debian_64', - :iso_file => "debian-7.5.0-amd64-netinst.iso", - :iso_src => "http://cdimage.debian.org/mirror/cdimage/archive/7.5.0/amd64/iso-cd/debian-7.5.0-amd64-netinst.iso", - :iso_md5 => "8fdb6715228ea90faba58cb84644d296", + :iso_file => "debian-7.6.0-amd64-netinst.iso", + :iso_src => "http://cdimage.debian.org/debian-cd/7.6.0/amd64/iso-cd/debian-7.6.0-amd64-netinst.iso", + :iso_md5 => "8a3c2ad7fd7a9c4c7e9bcb5cae38c135", :iso_download_timeout => "1000", :boot_wait => "10", :boot_cmd_sequence => [ '', diff --git a/tools/appliance/definitions/systemvmtemplate/definition.rb b/tools/appliance/definitions/systemvmtemplate/definition.rb index d29efcd8fbe..6173f419a70 100644 --- a/tools/appliance/definitions/systemvmtemplate/definition.rb +++ b/tools/appliance/definitions/systemvmtemplate/definition.rb @@ -10,9 +10,9 @@ Veewee::Definition.declare({ :memory_size=> '256', :disk_size => '2500', :disk_format => 'VDI', :hostiocache => 'off', :os_type_id => 'Debian', - :iso_file => "debian-7.5.0-i386-netinst.iso", - :iso_src => "http://cdimage.debian.org/mirror/cdimage/archive/7.5.0/i386/iso-cd/debian-7.5.0-i386-netinst.iso", - :iso_md5 => "0e6eaacb5a5828473afe90f6df9c8f16", + :iso_file => "debian-7.6.0-i386-netinst.iso", + :iso_src => "http://cdimage.debian.org/debian-cd/7.6.0/i386/iso-cd/debian-7.6.0-i386-netinst.iso", + :iso_md5 => "528e1a7315da1bbf50bd4d187880a519", :iso_download_timeout => "1000", :boot_wait => "10", :boot_cmd_sequence => [ '', From e2f1740de59032087c9761f0586655bcf5ffbb06 Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 09:55:37 +0200 Subject: [PATCH 02/60] CLOUDSTACK-7143: split base.sh into its two functions. --- .../systemvm64template/apt_upgrade.sh | 8 +++++ .../definitions/systemvm64template/base.sh | 29 ------------------- .../systemvm64template/configure_grub.sh | 13 +++++++++ .../systemvm64template/definition.rb | 3 +- .../systemvmtemplate/apt_upgrade.sh | 8 +++++ .../definitions/systemvmtemplate/base.sh | 29 ------------------- .../systemvmtemplate/configure_grub.sh | 13 +++++++++ .../systemvmtemplate/definition.rb | 3 +- 8 files changed, 46 insertions(+), 60 deletions(-) create mode 100644 tools/appliance/definitions/systemvm64template/apt_upgrade.sh delete mode 100644 tools/appliance/definitions/systemvm64template/base.sh create mode 100644 tools/appliance/definitions/systemvm64template/configure_grub.sh create mode 100644 tools/appliance/definitions/systemvmtemplate/apt_upgrade.sh delete mode 100644 tools/appliance/definitions/systemvmtemplate/base.sh create mode 100644 tools/appliance/definitions/systemvmtemplate/configure_grub.sh diff --git a/tools/appliance/definitions/systemvm64template/apt_upgrade.sh b/tools/appliance/definitions/systemvm64template/apt_upgrade.sh new file mode 100644 index 00000000000..69acc1a9c6b --- /dev/null +++ b/tools/appliance/definitions/systemvm64template/apt_upgrade.sh @@ -0,0 +1,8 @@ +# Update the box + +export DEBIAN_FRONTEND=noninteractive +export DEBIAN_PRIORITY=critical + +apt-get -q -y --force-yes update +apt-get -q -y --force-yes install curl unzip +apt-get clean diff --git a/tools/appliance/definitions/systemvm64template/base.sh b/tools/appliance/definitions/systemvm64template/base.sh deleted file mode 100644 index 8166081cdce..00000000000 --- a/tools/appliance/definitions/systemvm64template/base.sh +++ /dev/null @@ -1,29 +0,0 @@ -# Update the box - -export DEBIAN_FRONTEND=noninteractive -export DEBIAN_PRIORITY=critical - -apt-get -q -y --force-yes update -apt-get -q -y --force-yes install curl unzip -apt-get clean - -# Set up sudo, TODO: Check security concerns -echo 'vagrant ALL=NOPASSWD:/bin/chmod, /bin/cp, /bin/mkdir, /bin/mount, /bin/umount' > /etc/sudoers.d/vagrant -echo 'cloud ALL=NOPASSWD:/bin/chmod, /bin/cp, /bin/mkdir, /bin/mount, /bin/umount' > /etc/sudoers.d/cloud - -# Tweak sshd to prevent DNS resolution (speed up logins) -echo 'UseDNS no' >> /etc/ssh/sshd_config - -# Remove 5s grub timeout to speed up booting -cat < /etc/default/grub -# If you change this file, run 'update-grub' afterwards to update -# /boot/grub/grub.cfg. - -GRUB_DEFAULT=0 -GRUB_TIMEOUT=0 -GRUB_DISTRIBUTOR=`lsb_release -i -s 2> /dev/null || echo Debian` -GRUB_CMDLINE_LINUX_DEFAULT="quiet" -GRUB_CMDLINE_LINUX="debian-installer=en_US" -EOF - -update-grub diff --git a/tools/appliance/definitions/systemvm64template/configure_grub.sh b/tools/appliance/definitions/systemvm64template/configure_grub.sh new file mode 100644 index 00000000000..d530bb93bf9 --- /dev/null +++ b/tools/appliance/definitions/systemvm64template/configure_grub.sh @@ -0,0 +1,13 @@ +# Remove 5s grub timeout to speed up booting +cat < /etc/default/grub +# If you change this file, run 'update-grub' afterwards to update +# /boot/grub/grub.cfg. + +GRUB_DEFAULT=0 +GRUB_TIMEOUT=0 +GRUB_DISTRIBUTOR=`lsb_release -i -s 2> /dev/null || echo Debian` +GRUB_CMDLINE_LINUX_DEFAULT="quiet" +GRUB_CMDLINE_LINUX="debian-installer=en_US" +EOF + +update-grub diff --git a/tools/appliance/definitions/systemvm64template/definition.rb b/tools/appliance/definitions/systemvm64template/definition.rb index 2aaac04fdd2..de776f38bfb 100644 --- a/tools/appliance/definitions/systemvm64template/definition.rb +++ b/tools/appliance/definitions/systemvm64template/definition.rb @@ -43,7 +43,8 @@ Veewee::Definition.declare({ :sudo_cmd => "echo '%p'|sudo -S sh '%f'", :shutdown_cmd => "halt -p", :postinstall_files => [ - "base.sh", + "apt_upgrade.sh", + "configure_grub.sh", "postinstall.sh", "cleanup.sh", "zerodisk.sh" diff --git a/tools/appliance/definitions/systemvmtemplate/apt_upgrade.sh b/tools/appliance/definitions/systemvmtemplate/apt_upgrade.sh new file mode 100644 index 00000000000..69acc1a9c6b --- /dev/null +++ b/tools/appliance/definitions/systemvmtemplate/apt_upgrade.sh @@ -0,0 +1,8 @@ +# Update the box + +export DEBIAN_FRONTEND=noninteractive +export DEBIAN_PRIORITY=critical + +apt-get -q -y --force-yes update +apt-get -q -y --force-yes install curl unzip +apt-get clean diff --git a/tools/appliance/definitions/systemvmtemplate/base.sh b/tools/appliance/definitions/systemvmtemplate/base.sh deleted file mode 100644 index 8166081cdce..00000000000 --- a/tools/appliance/definitions/systemvmtemplate/base.sh +++ /dev/null @@ -1,29 +0,0 @@ -# Update the box - -export DEBIAN_FRONTEND=noninteractive -export DEBIAN_PRIORITY=critical - -apt-get -q -y --force-yes update -apt-get -q -y --force-yes install curl unzip -apt-get clean - -# Set up sudo, TODO: Check security concerns -echo 'vagrant ALL=NOPASSWD:/bin/chmod, /bin/cp, /bin/mkdir, /bin/mount, /bin/umount' > /etc/sudoers.d/vagrant -echo 'cloud ALL=NOPASSWD:/bin/chmod, /bin/cp, /bin/mkdir, /bin/mount, /bin/umount' > /etc/sudoers.d/cloud - -# Tweak sshd to prevent DNS resolution (speed up logins) -echo 'UseDNS no' >> /etc/ssh/sshd_config - -# Remove 5s grub timeout to speed up booting -cat < /etc/default/grub -# If you change this file, run 'update-grub' afterwards to update -# /boot/grub/grub.cfg. - -GRUB_DEFAULT=0 -GRUB_TIMEOUT=0 -GRUB_DISTRIBUTOR=`lsb_release -i -s 2> /dev/null || echo Debian` -GRUB_CMDLINE_LINUX_DEFAULT="quiet" -GRUB_CMDLINE_LINUX="debian-installer=en_US" -EOF - -update-grub diff --git a/tools/appliance/definitions/systemvmtemplate/configure_grub.sh b/tools/appliance/definitions/systemvmtemplate/configure_grub.sh new file mode 100644 index 00000000000..d530bb93bf9 --- /dev/null +++ b/tools/appliance/definitions/systemvmtemplate/configure_grub.sh @@ -0,0 +1,13 @@ +# Remove 5s grub timeout to speed up booting +cat < /etc/default/grub +# If you change this file, run 'update-grub' afterwards to update +# /boot/grub/grub.cfg. + +GRUB_DEFAULT=0 +GRUB_TIMEOUT=0 +GRUB_DISTRIBUTOR=`lsb_release -i -s 2> /dev/null || echo Debian` +GRUB_CMDLINE_LINUX_DEFAULT="quiet" +GRUB_CMDLINE_LINUX="debian-installer=en_US" +EOF + +update-grub diff --git a/tools/appliance/definitions/systemvmtemplate/definition.rb b/tools/appliance/definitions/systemvmtemplate/definition.rb index 6173f419a70..d2e25df2d53 100644 --- a/tools/appliance/definitions/systemvmtemplate/definition.rb +++ b/tools/appliance/definitions/systemvmtemplate/definition.rb @@ -43,7 +43,8 @@ Veewee::Definition.declare({ :sudo_cmd => "echo '%p'|sudo -S sh '%f'", :shutdown_cmd => "halt -p", :postinstall_files => [ - "base.sh", + "apt_upgrade.sh", + "configure_grub.sh", "postinstall.sh", "cleanup.sh", "zerodisk.sh" From 262cde5fb7eaf2125c51617ed614ac4fefac9d25 Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 09:57:49 +0200 Subject: [PATCH 03/60] CLOUDSTACK-7143: move network tuning from cleanup.sh to its own script. --- tools/appliance/definitions/systemvm64template/cleanup.sh | 4 ---- .../definitions/systemvm64template/configure_networking.sh | 2 ++ tools/appliance/definitions/systemvm64template/definition.rb | 1 + tools/appliance/definitions/systemvmtemplate/cleanup.sh | 4 ---- .../definitions/systemvmtemplate/configure_networking.sh | 2 ++ tools/appliance/definitions/systemvmtemplate/definition.rb | 1 + 6 files changed, 6 insertions(+), 8 deletions(-) create mode 100644 tools/appliance/definitions/systemvm64template/configure_networking.sh create mode 100644 tools/appliance/definitions/systemvmtemplate/configure_networking.sh diff --git a/tools/appliance/definitions/systemvm64template/cleanup.sh b/tools/appliance/definitions/systemvm64template/cleanup.sh index 701d8d84000..3b1d4bf9f88 100644 --- a/tools/appliance/definitions/systemvm64template/cleanup.sh +++ b/tools/appliance/definitions/systemvm64template/cleanup.sh @@ -14,7 +14,3 @@ echo "cleaning up udev rules" rm /etc/udev/rules.d/70-persistent-net.rules rm -rf /dev/.udev/ rm /lib/udev/rules.d/75-persistent-net-generator.rules - -echo "Adding a 2 sec delay to the interface up, to make the dhclient happy" -echo "pre-up sleep 2" >> /etc/network/interfaces - diff --git a/tools/appliance/definitions/systemvm64template/configure_networking.sh b/tools/appliance/definitions/systemvm64template/configure_networking.sh new file mode 100644 index 00000000000..eca0d5c1c45 --- /dev/null +++ b/tools/appliance/definitions/systemvm64template/configure_networking.sh @@ -0,0 +1,2 @@ +echo "Adding a 2 sec delay to the interface up, to make the dhclient happy" +echo "pre-up sleep 2" >> /etc/network/interfaces diff --git a/tools/appliance/definitions/systemvm64template/definition.rb b/tools/appliance/definitions/systemvm64template/definition.rb index de776f38bfb..d1aeb28972e 100644 --- a/tools/appliance/definitions/systemvm64template/definition.rb +++ b/tools/appliance/definitions/systemvm64template/definition.rb @@ -47,6 +47,7 @@ Veewee::Definition.declare({ "configure_grub.sh", "postinstall.sh", "cleanup.sh", + "configure_networking.sh", "zerodisk.sh" ], :postinstall_timeout => "10000" diff --git a/tools/appliance/definitions/systemvmtemplate/cleanup.sh b/tools/appliance/definitions/systemvmtemplate/cleanup.sh index 701d8d84000..3b1d4bf9f88 100644 --- a/tools/appliance/definitions/systemvmtemplate/cleanup.sh +++ b/tools/appliance/definitions/systemvmtemplate/cleanup.sh @@ -14,7 +14,3 @@ echo "cleaning up udev rules" rm /etc/udev/rules.d/70-persistent-net.rules rm -rf /dev/.udev/ rm /lib/udev/rules.d/75-persistent-net-generator.rules - -echo "Adding a 2 sec delay to the interface up, to make the dhclient happy" -echo "pre-up sleep 2" >> /etc/network/interfaces - diff --git a/tools/appliance/definitions/systemvmtemplate/configure_networking.sh b/tools/appliance/definitions/systemvmtemplate/configure_networking.sh new file mode 100644 index 00000000000..eca0d5c1c45 --- /dev/null +++ b/tools/appliance/definitions/systemvmtemplate/configure_networking.sh @@ -0,0 +1,2 @@ +echo "Adding a 2 sec delay to the interface up, to make the dhclient happy" +echo "pre-up sleep 2" >> /etc/network/interfaces diff --git a/tools/appliance/definitions/systemvmtemplate/definition.rb b/tools/appliance/definitions/systemvmtemplate/definition.rb index d2e25df2d53..21210f06ec8 100644 --- a/tools/appliance/definitions/systemvmtemplate/definition.rb +++ b/tools/appliance/definitions/systemvmtemplate/definition.rb @@ -47,6 +47,7 @@ Veewee::Definition.declare({ "configure_grub.sh", "postinstall.sh", "cleanup.sh", + "configure_networking.sh", "zerodisk.sh" ], :postinstall_timeout => "10000" From 039a47bd5b85d4860ddc482c67065f5187876056 Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 10:36:11 +0200 Subject: [PATCH 04/60] CLOUDSTACK-7143: merge systemvm templates, step 1: remove differences --- .../systemvm64template/definition.rb | 111 ++++++++++-------- .../systemvm64template/postinstall.sh | 25 ++-- .../systemvm64template/preseed.cfg | 4 +- .../systemvmtemplate/definition.rb | 111 ++++++++++-------- .../systemvmtemplate/postinstall.sh | 22 +++- .../definitions/systemvmtemplate/preseed.cfg | 5 +- 6 files changed, 168 insertions(+), 110 deletions(-) diff --git a/tools/appliance/definitions/systemvm64template/definition.rb b/tools/appliance/definitions/systemvm64template/definition.rb index d1aeb28972e..594b7a8b47c 100644 --- a/tools/appliance/definitions/systemvm64template/definition.rb +++ b/tools/appliance/definitions/systemvm64template/definition.rb @@ -1,3 +1,5 @@ +arch = ENV['VM_ARCH'] || 'amd64' + # # NOTE: Before changing the version of the debian image make # sure it is added to the userContent of jenkins.buildacloud.org @@ -5,50 +7,65 @@ # This will prevent the inevitable build failure once the iso is # removed from the debian mirrors # -Veewee::Definition.declare({ - :cpu_count => '1', - :memory_size=> '256', - :disk_size => '2500', :disk_format => 'VDI', :hostiocache => 'off', - :os_type_id => 'Debian_64', - :iso_file => "debian-7.6.0-amd64-netinst.iso", - :iso_src => "http://cdimage.debian.org/debian-cd/7.6.0/amd64/iso-cd/debian-7.6.0-amd64-netinst.iso", - :iso_md5 => "8a3c2ad7fd7a9c4c7e9bcb5cae38c135", - :iso_download_timeout => "1000", - :boot_wait => "10", :boot_cmd_sequence => [ - '', - 'install ', - 'preseed/url=http://%IP%:%PORT%/preseed.cfg ', - 'debian-installer=en_US ', - 'auto ', - 'locale=en_US ', - 'kbd-chooser/method=us ', - 'netcfg/get_hostname=systemvm ', - 'netcfg/get_domain=apache.org ', - 'fb=false ', - 'debconf/frontend=noninteractive ', - 'console-setup/ask_detect=false ', - 'console-keymaps-at/keymap=us ', - 'keyboard-configuration/xkb-keymap=us ', - '' - ], - :kickstart_port => "7122", - :kickstart_timeout => "10000", - :kickstart_file => "preseed.cfg", - :ssh_login_timeout => "10000", - :ssh_user => "root", - :ssh_password => "password", - :ssh_key => "", - :ssh_host_port => "7222", - :ssh_guest_port => "22", - :sudo_cmd => "echo '%p'|sudo -S sh '%f'", - :shutdown_cmd => "halt -p", - :postinstall_files => [ - "apt_upgrade.sh", - "configure_grub.sh", - "postinstall.sh", - "cleanup.sh", - "configure_networking.sh", - "zerodisk.sh" - ], - :postinstall_timeout => "10000" -}) +architectures = { + :i386 => { + :os_type_id => 'Debian', + :iso_file => "debian-7.6.0-i386-netinst.iso", + :iso_src => "http://cdimage.debian.org/debian-cd/7.6.0/i386/iso-cd/debian-7.6.0-i386-netinst.iso", + :iso_md5 => "528e1a7315da1bbf50bd4d187880a519", + }, + :amd64 => { + :os_type_id => 'Debian_64', + :iso_file => 'debian-7.6.0-amd64-netinst.iso', + :iso_src => 'http://cdimage.debian.org/debian-cd/7.6.0/amd64/iso-cd/debian-7.6.0-amd64-netinst.iso', + :iso_md5 => '8a3c2ad7fd7a9c4c7e9bcb5cae38c135' + } +} + +config = { + :cpu_count => '1', + :memory_size=> '256', + :disk_size => '2500', :disk_format => 'VDI', :hostiocache => 'off', + :iso_download_timeout => "1000", + :boot_wait => "10", :boot_cmd_sequence => [ + '', + 'install ', + 'preseed/url=http://%IP%:%PORT%/preseed.cfg ', + 'debian-installer=en_US ', + 'auto ', + 'locale=en_US ', + 'kbd-chooser/method=us ', + 'netcfg/get_hostname=systemvm ', + 'netcfg/get_domain=apache.org ', + 'fb=false ', + 'debconf/frontend=noninteractive ', + 'console-setup/ask_detect=false ', + 'console-keymaps-at/keymap=us ', + 'keyboard-configuration/xkb-keymap=us ', + '' + ], + :kickstart_port => "7122", + :kickstart_timeout => "10000", + :kickstart_file => "preseed.cfg", + :ssh_login_timeout => "10000", + :ssh_user => "root", + :ssh_password => "password", + :ssh_key => "", + :ssh_host_port => "7222", + :ssh_guest_port => "22", + :sudo_cmd => "echo '%p'|sudo -S sh '%f'", + :shutdown_cmd => "halt -p", + :postinstall_files => [ + "apt_upgrade.sh", + "configure_grub.sh", + "postinstall.sh", + "cleanup.sh", + "configure_networking.sh", + "zerodisk.sh" + ], + :postinstall_timeout => "10000" +} + +config.merge! architectures[arch.to_sym] + +Veewee::Definition.declare(config) diff --git a/tools/appliance/definitions/systemvm64template/postinstall.sh b/tools/appliance/definitions/systemvm64template/postinstall.sh index 32e59d91b88..d246bbdb2b4 100644 --- a/tools/appliance/definitions/systemvm64template/postinstall.sh +++ b/tools/appliance/definitions/systemvm64template/postinstall.sh @@ -30,6 +30,7 @@ add_backports () { install_packages() { DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical + local arch=`dpkg --print-architecture` # Basic packages apt-get --no-install-recommends -q -y --force-yes install rsyslog logrotate cron chkconfig insserv net-tools ifupdown vim-tiny netbase iptables @@ -76,11 +77,13 @@ install_packages() { apt-get --no-install-recommends -q -y --force-yes install iptables-persistent # Hyperv kvp daemon - 64bit only - # Download the hv kvp daemon - wget http://people.apache.org/~rajeshbattala/hv-kvp-daemon_3.1_amd64.deb - dpkg -i hv-kvp-daemon_3.1_amd64.deb + if [ "${arch}" == "amd64" ]; then + # Download the hv kvp daemon + wget http://people.apache.org/~rajeshbattala/hv-kvp-daemon_3.1_amd64.deb + dpkg -i hv-kvp-daemon_3.1_amd64.deb + fi - #libraries required for rdp client (Hyper-V) + #libraries required for rdp client (Hyper-V) apt-get --no-install-recommends -q -y --force-yes install libtcnative-1 libssl-dev libapr1-dev # vmware tools @@ -102,9 +105,11 @@ install_packages() { apt-get --no-install-recommends -q -y --force-yes install haproxy #32 bit architecture support:: not required for 32 bit template - dpkg --add-architecture i386 - apt-get update - apt-get --no-install-recommends -q -y --force-yes install links:i386 libuuid1:i386 + if [ "${arch}" != "i386" ]; then + dpkg --add-architecture i386 + apt-get update + apt-get --no-install-recommends -q -y --force-yes install links:i386 libuuid1:i386 + fi apt-get --no-install-recommends -q -y --force-yes install radvd } @@ -226,6 +231,7 @@ configure_apache2() { } configure_services() { + local arch=`dpkg --print-architecture` mkdir -p /var/www/html mkdir -p /opt/cloud/bin mkdir -p /var/cache/cloud @@ -258,7 +264,10 @@ configure_services() { chkconfig --add cloud chkconfig cloud off chkconfig xl2tpd off - chkconfig hv_kvp_daemon off + # Hyperv kvp daemon - 64bit only + if [ "${arch}" == "amd64" ]; then + chkconfig hv_kvp_daemon off + fi chkconfig radvd off } diff --git a/tools/appliance/definitions/systemvm64template/preseed.cfg b/tools/appliance/definitions/systemvm64template/preseed.cfg index d11b625e44d..6a10e777de9 100644 --- a/tools/appliance/definitions/systemvm64template/preseed.cfg +++ b/tools/appliance/definitions/systemvm64template/preseed.cfg @@ -334,11 +334,9 @@ d-i finish-install/reboot_in_progress note # debconf-get-selections --installer > file # debconf-get-selections >> file -libssl1.0.0 libssl1.0.0/restart-services string -libssl1.0.0:amd64 libssl1.0.0/restart-services string +libssl1.0.0 libssl1.0.0/restart-services string libssl1.0.0 libssl1.0.0/restart-failed error -libssl1.0.0:amd64 libssl1.0.0/restart-failed error #### Advanced options ### Running custom commands during the installation diff --git a/tools/appliance/definitions/systemvmtemplate/definition.rb b/tools/appliance/definitions/systemvmtemplate/definition.rb index 21210f06ec8..c57bf237b33 100644 --- a/tools/appliance/definitions/systemvmtemplate/definition.rb +++ b/tools/appliance/definitions/systemvmtemplate/definition.rb @@ -1,3 +1,5 @@ +arch = ENV['VM_ARCH'] || 'i386' + # # NOTE: Before changing the version of the debian image make # sure it is added to the userContent of jenkins.buildacloud.org @@ -5,50 +7,65 @@ # This will prevent the inevitable build failure once the iso is # removed from the debian mirrors # -Veewee::Definition.declare({ - :cpu_count => '1', - :memory_size=> '256', - :disk_size => '2500', :disk_format => 'VDI', :hostiocache => 'off', - :os_type_id => 'Debian', - :iso_file => "debian-7.6.0-i386-netinst.iso", - :iso_src => "http://cdimage.debian.org/debian-cd/7.6.0/i386/iso-cd/debian-7.6.0-i386-netinst.iso", - :iso_md5 => "528e1a7315da1bbf50bd4d187880a519", - :iso_download_timeout => "1000", - :boot_wait => "10", :boot_cmd_sequence => [ - '', - 'install ', - 'preseed/url=http://%IP%:%PORT%/preseed.cfg ', - 'debian-installer=en_US ', - 'auto ', - 'locale=en_US ', - 'kbd-chooser/method=us ', - 'netcfg/get_hostname=systemvm ', - 'netcfg/get_domain=apache.org ', - 'fb=false ', - 'debconf/frontend=noninteractive ', - 'console-setup/ask_detect=false ', - 'console-keymaps-at/keymap=us ', - 'keyboard-configuration/xkb-keymap=us ', - '' - ], - :kickstart_port => "7122", - :kickstart_timeout => "10000", - :kickstart_file => "preseed.cfg", - :ssh_login_timeout => "10000", - :ssh_user => "root", - :ssh_password => "password", - :ssh_key => "", - :ssh_host_port => "7222", - :ssh_guest_port => "22", - :sudo_cmd => "echo '%p'|sudo -S sh '%f'", - :shutdown_cmd => "halt -p", - :postinstall_files => [ - "apt_upgrade.sh", - "configure_grub.sh", - "postinstall.sh", - "cleanup.sh", - "configure_networking.sh", - "zerodisk.sh" - ], - :postinstall_timeout => "10000" -}) +architectures = { + :i386 => { + :os_type_id => 'Debian', + :iso_file => "debian-7.6.0-i386-netinst.iso", + :iso_src => "http://cdimage.debian.org/debian-cd/7.6.0/i386/iso-cd/debian-7.6.0-i386-netinst.iso", + :iso_md5 => "528e1a7315da1bbf50bd4d187880a519", + }, + :amd64 => { + :os_type_id => 'Debian_64', + :iso_file => 'debian-7.6.0-amd64-netinst.iso', + :iso_src => 'http://cdimage.debian.org/debian-cd/7.6.0/amd64/iso-cd/debian-7.6.0-amd64-netinst.iso', + :iso_md5 => '8a3c2ad7fd7a9c4c7e9bcb5cae38c135' + } +} + +config = { + :cpu_count => '1', + :memory_size=> '256', + :disk_size => '2500', :disk_format => 'VDI', :hostiocache => 'off', + :iso_download_timeout => "1000", + :boot_wait => "10", :boot_cmd_sequence => [ + '', + 'install ', + 'preseed/url=http://%IP%:%PORT%/preseed.cfg ', + 'debian-installer=en_US ', + 'auto ', + 'locale=en_US ', + 'kbd-chooser/method=us ', + 'netcfg/get_hostname=systemvm ', + 'netcfg/get_domain=apache.org ', + 'fb=false ', + 'debconf/frontend=noninteractive ', + 'console-setup/ask_detect=false ', + 'console-keymaps-at/keymap=us ', + 'keyboard-configuration/xkb-keymap=us ', + '' + ], + :kickstart_port => "7122", + :kickstart_timeout => "10000", + :kickstart_file => "preseed.cfg", + :ssh_login_timeout => "10000", + :ssh_user => "root", + :ssh_password => "password", + :ssh_key => "", + :ssh_host_port => "7222", + :ssh_guest_port => "22", + :sudo_cmd => "echo '%p'|sudo -S sh '%f'", + :shutdown_cmd => "halt -p", + :postinstall_files => [ + "apt_upgrade.sh", + "configure_grub.sh", + "postinstall.sh", + "cleanup.sh", + "configure_networking.sh", + "zerodisk.sh" + ], + :postinstall_timeout => "10000" +} + +config.merge! architectures[arch.to_sym] + +Veewee::Definition.declare(config) diff --git a/tools/appliance/definitions/systemvmtemplate/postinstall.sh b/tools/appliance/definitions/systemvmtemplate/postinstall.sh index a292203c7fc..e00d259ec8a 100644 --- a/tools/appliance/definitions/systemvmtemplate/postinstall.sh +++ b/tools/appliance/definitions/systemvmtemplate/postinstall.sh @@ -30,6 +30,7 @@ add_backports () { install_packages() { DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical + local arch=`dpkg --print-architecture` # Basic packages apt-get --no-install-recommends -q -y --force-yes install rsyslog logrotate cron chkconfig insserv net-tools ifupdown vim-tiny netbase iptables @@ -75,7 +76,14 @@ install_packages() { echo "iptables-persistent iptables-persistent/autosave_v6 boolean true" | debconf-set-selections apt-get --no-install-recommends -q -y --force-yes install iptables-persistent - #libraries required for rdp client (Hyper-V) + # Hyperv kvp daemon - 64bit only + if [ "${arch}" == "amd64" ]; then + # Download the hv kvp daemon + wget http://people.apache.org/~rajeshbattala/hv-kvp-daemon_3.1_amd64.deb + dpkg -i hv-kvp-daemon_3.1_amd64.deb + fi + + #libraries required for rdp client (Hyper-V) apt-get --no-install-recommends -q -y --force-yes install libtcnative-1 libssl-dev libapr1-dev # vmware tools @@ -96,6 +104,13 @@ install_packages() { apt-get --no-install-recommends -q -y --force-yes install haproxy + #32 bit architecture support:: not required for 32 bit template + if [ "${arch}" != "i386" ]; then + dpkg --add-architecture i386 + apt-get update + apt-get --no-install-recommends -q -y --force-yes install links:i386 libuuid1:i386 + fi + apt-get --no-install-recommends -q -y --force-yes install radvd } @@ -216,6 +231,7 @@ configure_apache2() { } configure_services() { + local arch=`dpkg --print-architecture` mkdir -p /var/www/html mkdir -p /opt/cloud/bin mkdir -p /var/cache/cloud @@ -248,6 +264,10 @@ configure_services() { chkconfig --add cloud chkconfig cloud off chkconfig xl2tpd off + # Hyperv kvp daemon - 64bit only + if [ "${arch}" == "amd64" ]; then + chkconfig hv_kvp_daemon off + fi chkconfig radvd off } diff --git a/tools/appliance/definitions/systemvmtemplate/preseed.cfg b/tools/appliance/definitions/systemvmtemplate/preseed.cfg index 1e4fb4ef8c4..6a10e777de9 100644 --- a/tools/appliance/definitions/systemvmtemplate/preseed.cfg +++ b/tools/appliance/definitions/systemvmtemplate/preseed.cfg @@ -334,12 +334,9 @@ d-i finish-install/reboot_in_progress note # debconf-get-selections --installer > file # debconf-get-selections >> file - -libssl1.0.0 libssl1.0.0/restart-services string -libssl1.0.0:i386 libssl1.0.0/restart-services string +libssl1.0.0 libssl1.0.0/restart-services string libssl1.0.0 libssl1.0.0/restart-failed error -libssl1.0.0:i386 libssl1.0.0/restart-failed error #### Advanced options ### Running custom commands during the installation From 3c0cdb595f44281e70f7bb439b67072f569a2d60 Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 10:44:38 +0200 Subject: [PATCH 05/60] CLOUDSTACK-7143: merge systemvm templates, step 2: only one template remains --- tools/appliance/README.md | 6 + tools/appliance/build.sh | 12 +- .../systemvm64template/apt_upgrade.sh | 8 - .../definitions/systemvm64template/cleanup.sh | 16 - .../systemvm64template/configure_grub.sh | 13 - .../configure_networking.sh | 2 - .../systemvm64template/definition.rb | 71 ---- .../systemvm64template/postinstall.sh | 300 --------------- .../systemvm64template/preseed.cfg | 360 ------------------ .../systemvm64template/zerodisk.sh | 15 - 10 files changed, 13 insertions(+), 790 deletions(-) delete mode 100644 tools/appliance/definitions/systemvm64template/apt_upgrade.sh delete mode 100644 tools/appliance/definitions/systemvm64template/cleanup.sh delete mode 100644 tools/appliance/definitions/systemvm64template/configure_grub.sh delete mode 100644 tools/appliance/definitions/systemvm64template/configure_networking.sh delete mode 100644 tools/appliance/definitions/systemvm64template/definition.rb delete mode 100644 tools/appliance/definitions/systemvm64template/postinstall.sh delete mode 100644 tools/appliance/definitions/systemvm64template/preseed.cfg delete mode 100644 tools/appliance/definitions/systemvm64template/zerodisk.sh diff --git a/tools/appliance/README.md b/tools/appliance/README.md index ffdf6b22d8b..8c51c9ca756 100644 --- a/tools/appliance/README.md +++ b/tools/appliance/README.md @@ -68,6 +68,12 @@ Halt the box: Now VirtualBox can be used to export appliance. +To build the systemvm64template by hand using veewee, set VM_ARCH=amd64 and use +the systemvmtemplate: + + export VM_ARCH=amd64 + cp -r definitions/systemvmtemplate definitions/systemvm64template + veewee vbox build 'systemvm64template' Trobuleshooting =============== diff --git a/tools/appliance/build.sh b/tools/appliance/build.sh index 29d92caf172..57d2298bd70 100755 --- a/tools/appliance/build.sh +++ b/tools/appliance/build.sh @@ -18,11 +18,13 @@ set -x -if [ ! -z "$1" ] -then - appliance="$1" -else - appliance="systemvmtemplate" +appliance="${1:-systemvmtemplate}" +arch="i386" +if [ "${appliance}" == "systemvm64template" ]; then + arch="amd64" + export VM_ARCH="${arch}" + rm -rf definitions/systemvm64template + cp -r definitions/systemvmtemplate definitions/systemvm64template fi build_date=`date +%Y-%m-%d` diff --git a/tools/appliance/definitions/systemvm64template/apt_upgrade.sh b/tools/appliance/definitions/systemvm64template/apt_upgrade.sh deleted file mode 100644 index 69acc1a9c6b..00000000000 --- a/tools/appliance/definitions/systemvm64template/apt_upgrade.sh +++ /dev/null @@ -1,8 +0,0 @@ -# Update the box - -export DEBIAN_FRONTEND=noninteractive -export DEBIAN_PRIORITY=critical - -apt-get -q -y --force-yes update -apt-get -q -y --force-yes install curl unzip -apt-get clean diff --git a/tools/appliance/definitions/systemvm64template/cleanup.sh b/tools/appliance/definitions/systemvm64template/cleanup.sh deleted file mode 100644 index 3b1d4bf9f88..00000000000 --- a/tools/appliance/definitions/systemvm64template/cleanup.sh +++ /dev/null @@ -1,16 +0,0 @@ -# Clean up -#apt-get -y remove linux-headers-$(uname -r) build-essential -apt-get -y remove dictionaries-common busybox -apt-get -y autoremove -apt-get autoclean -apt-get clean - -# Removing leftover leases and persistent rules -echo "cleaning up dhcp leases" -rm /var/lib/dhcp/* - -# Make sure Udev doesn't block our network -echo "cleaning up udev rules" -rm /etc/udev/rules.d/70-persistent-net.rules -rm -rf /dev/.udev/ -rm /lib/udev/rules.d/75-persistent-net-generator.rules diff --git a/tools/appliance/definitions/systemvm64template/configure_grub.sh b/tools/appliance/definitions/systemvm64template/configure_grub.sh deleted file mode 100644 index d530bb93bf9..00000000000 --- a/tools/appliance/definitions/systemvm64template/configure_grub.sh +++ /dev/null @@ -1,13 +0,0 @@ -# Remove 5s grub timeout to speed up booting -cat < /etc/default/grub -# If you change this file, run 'update-grub' afterwards to update -# /boot/grub/grub.cfg. - -GRUB_DEFAULT=0 -GRUB_TIMEOUT=0 -GRUB_DISTRIBUTOR=`lsb_release -i -s 2> /dev/null || echo Debian` -GRUB_CMDLINE_LINUX_DEFAULT="quiet" -GRUB_CMDLINE_LINUX="debian-installer=en_US" -EOF - -update-grub diff --git a/tools/appliance/definitions/systemvm64template/configure_networking.sh b/tools/appliance/definitions/systemvm64template/configure_networking.sh deleted file mode 100644 index eca0d5c1c45..00000000000 --- a/tools/appliance/definitions/systemvm64template/configure_networking.sh +++ /dev/null @@ -1,2 +0,0 @@ -echo "Adding a 2 sec delay to the interface up, to make the dhclient happy" -echo "pre-up sleep 2" >> /etc/network/interfaces diff --git a/tools/appliance/definitions/systemvm64template/definition.rb b/tools/appliance/definitions/systemvm64template/definition.rb deleted file mode 100644 index 594b7a8b47c..00000000000 --- a/tools/appliance/definitions/systemvm64template/definition.rb +++ /dev/null @@ -1,71 +0,0 @@ -arch = ENV['VM_ARCH'] || 'amd64' - -# -# NOTE: Before changing the version of the debian image make -# sure it is added to the userContent of jenkins.buildacloud.org -# and the copy task is updated on the systemvm builds -# This will prevent the inevitable build failure once the iso is -# removed from the debian mirrors -# -architectures = { - :i386 => { - :os_type_id => 'Debian', - :iso_file => "debian-7.6.0-i386-netinst.iso", - :iso_src => "http://cdimage.debian.org/debian-cd/7.6.0/i386/iso-cd/debian-7.6.0-i386-netinst.iso", - :iso_md5 => "528e1a7315da1bbf50bd4d187880a519", - }, - :amd64 => { - :os_type_id => 'Debian_64', - :iso_file => 'debian-7.6.0-amd64-netinst.iso', - :iso_src => 'http://cdimage.debian.org/debian-cd/7.6.0/amd64/iso-cd/debian-7.6.0-amd64-netinst.iso', - :iso_md5 => '8a3c2ad7fd7a9c4c7e9bcb5cae38c135' - } -} - -config = { - :cpu_count => '1', - :memory_size=> '256', - :disk_size => '2500', :disk_format => 'VDI', :hostiocache => 'off', - :iso_download_timeout => "1000", - :boot_wait => "10", :boot_cmd_sequence => [ - '', - 'install ', - 'preseed/url=http://%IP%:%PORT%/preseed.cfg ', - 'debian-installer=en_US ', - 'auto ', - 'locale=en_US ', - 'kbd-chooser/method=us ', - 'netcfg/get_hostname=systemvm ', - 'netcfg/get_domain=apache.org ', - 'fb=false ', - 'debconf/frontend=noninteractive ', - 'console-setup/ask_detect=false ', - 'console-keymaps-at/keymap=us ', - 'keyboard-configuration/xkb-keymap=us ', - '' - ], - :kickstart_port => "7122", - :kickstart_timeout => "10000", - :kickstart_file => "preseed.cfg", - :ssh_login_timeout => "10000", - :ssh_user => "root", - :ssh_password => "password", - :ssh_key => "", - :ssh_host_port => "7222", - :ssh_guest_port => "22", - :sudo_cmd => "echo '%p'|sudo -S sh '%f'", - :shutdown_cmd => "halt -p", - :postinstall_files => [ - "apt_upgrade.sh", - "configure_grub.sh", - "postinstall.sh", - "cleanup.sh", - "configure_networking.sh", - "zerodisk.sh" - ], - :postinstall_timeout => "10000" -} - -config.merge! architectures[arch.to_sym] - -Veewee::Definition.declare(config) diff --git a/tools/appliance/definitions/systemvm64template/postinstall.sh b/tools/appliance/definitions/systemvm64template/postinstall.sh deleted file mode 100644 index d246bbdb2b4..00000000000 --- a/tools/appliance/definitions/systemvm64template/postinstall.sh +++ /dev/null @@ -1,300 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -x - -ROOTPW=password -HOSTNAME=systemvm -CLOUDSTACK_RELEASE=4.5.0 - -add_backports () { - sed -i '/backports/d' /etc/apt/sources.list - echo 'deb http://http.us.debian.org/debian wheezy-backports main' >> /etc/apt/sources.list - apt-get update -} - -install_packages() { - DEBIAN_FRONTEND=noninteractive - DEBIAN_PRIORITY=critical - local arch=`dpkg --print-architecture` - - # Basic packages - apt-get --no-install-recommends -q -y --force-yes install rsyslog logrotate cron chkconfig insserv net-tools ifupdown vim-tiny netbase iptables - apt-get --no-install-recommends -q -y --force-yes install openssh-server openssl e2fsprogs dhcp3-client tcpdump socat wget - # apt-get --no-install-recommends -q -y --force-yes install grub-legacy - apt-get --no-install-recommends -q -y --force-yes install python bzip2 sed gawk diffutils grep gzip less tar telnet ftp rsync traceroute psmisc lsof procps inetutils-ping iputils-arping httping - apt-get --no-install-recommends -q -y --force-yes install dnsutils zip unzip ethtool uuid file iproute acpid virt-what sudo - - # sysstat - echo 'sysstat sysstat/enable boolean true' | debconf-set-selections - apt-get --no-install-recommends -q -y --force-yes install sysstat - # apache - apt-get --no-install-recommends -q -y --force-yes install apache2 ssl-cert - - # dnsmasq - apt-get --no-install-recommends -q -y --force-yes install dnsmasq dnsmasq-utils - # nfs client - apt-get --no-install-recommends -q -y --force-yes install nfs-common - # nfs irqbalance - apt-get --no-install-recommends -q -y --force-yes install irqbalance - - # cifs client - apt-get --no-install-recommends -q -y --force-yes install samba-common - apt-get --no-install-recommends -q -y --force-yes install cifs-utils - - # vpn stuff - apt-get --no-install-recommends -q -y --force-yes install xl2tpd bcrelay ppp ipsec-tools tdb-tools - echo "openswan openswan/install_x509_certificate boolean false" | debconf-set-selections - echo "openswan openswan/install_x509_certificate seen true" | debconf-set-selections - apt-get --no-install-recommends -q -y --force-yes install openswan=1:2.6.37-3 - - # xenstore utils - apt-get --no-install-recommends -q -y --force-yes install xenstore-utils libxenstore3.0 - # keepalived and conntrackd for redundant router - apt-get --no-install-recommends -q -y --force-yes install keepalived conntrackd ipvsadm libnetfilter-conntrack3 libnl1 - # ipcalc - apt-get --no-install-recommends -q -y --force-yes install ipcalc - apt-get update - # java - apt-get --no-install-recommends -q -y --force-yes install openjdk-7-jre-headless - - echo "iptables-persistent iptables-persistent/autosave_v4 boolean true" | debconf-set-selections - echo "iptables-persistent iptables-persistent/autosave_v6 boolean true" | debconf-set-selections - apt-get --no-install-recommends -q -y --force-yes install iptables-persistent - - # Hyperv kvp daemon - 64bit only - if [ "${arch}" == "amd64" ]; then - # Download the hv kvp daemon - wget http://people.apache.org/~rajeshbattala/hv-kvp-daemon_3.1_amd64.deb - dpkg -i hv-kvp-daemon_3.1_amd64.deb - fi - - #libraries required for rdp client (Hyper-V) - apt-get --no-install-recommends -q -y --force-yes install libtcnative-1 libssl-dev libapr1-dev - - # vmware tools - apt-get --no-install-recommends -q -y --force-yes install open-vm-tools - # commented installaion of vmware-tools as we are using the opensource open-vm-tools: - # apt-get --no-install-recommends -q -y --force-yes install build-essential linux-headers-`uname -r` - # df -h - # PREVDIR=$PWD - # cd /opt - # wget http://people.apache.org/~bhaisaab/cloudstack/VMwareTools-9.2.1-818201.tar.gz - # tar xzf VMwareTools-9.2.1-818201.tar.gz - # rm VMwareTools-*.tar.gz - # cd vmware-tools-distrib - # ./vmware-install.pl -d - # cd $PREV - # rm -fr /opt/vmware-tools-distrib - # apt-get -q -y --force-yes purge build-essential - - apt-get --no-install-recommends -q -y --force-yes install haproxy - - #32 bit architecture support:: not required for 32 bit template - if [ "${arch}" != "i386" ]; then - dpkg --add-architecture i386 - apt-get update - apt-get --no-install-recommends -q -y --force-yes install links:i386 libuuid1:i386 - fi - - apt-get --no-install-recommends -q -y --force-yes install radvd -} - -setup_accounts() { - # Setup sudo to allow no-password sudo for "admin" - groupadd -r admin - # Create a 'cloud' user if it's not there - id cloud - if [[ $? -ne 0 ]] - then - useradd -G admin cloud - else - usermod -a -G admin cloud - fi - echo "root:$ROOTPW" | chpasswd - echo "cloud:`openssl rand -base64 32`" | chpasswd - sed -i -e '/Defaults\s\+env_reset/a Defaults\texempt_group=admin' /etc/sudoers - sed -i -e 's/%admin ALL=(ALL) ALL/%admin ALL=NOPASSWD:/bin/chmod, /bin/cp, /bin/mkdir, /bin/mount, /bin/umount/g' /etc/sudoers - # Disable password based authentication via ssh, this will take effect on next reboot - sed -i -e 's/^.*PasswordAuthentication .*$/PasswordAuthentication no/g' /etc/ssh/sshd_config - # Secure ~/.ssh - mkdir -p /home/cloud/.ssh - chmod 700 /home/cloud/.ssh -} - -fix_nameserver() { - # Replace /etc/resolv.conf also - cat > /etc/resolv.conf << EOF -nameserver 8.8.8.8 -nameserver 8.8.4.4 -EOF -} - -fix_inittab() { - # Fix inittab - cat >> /etc/inittab << EOF - -vc:2345:respawn:/sbin/getty 38400 hvc0 -EOF -} - -fix_acpid() { - # Fix acpid - mkdir -p /etc/acpi/events - cat >> /etc/acpi/events/power << EOF -event=button/power.* -action=/usr/local/sbin/power.sh "%e" -EOF - cat >> /usr/local/sbin/power.sh << EOF -#!/bin/bash -/sbin/poweroff -EOF - chmod a+x /usr/local/sbin/power.sh -} - -fix_hostname() { - # Fix hostname in openssh-server generated keys - sed -i "s/root@\(.*\)$/root@$HOSTNAME/g" /etc/ssh/ssh_host_*.pub - # Fix hostname to override one provided by dhcp during vm build - echo "$HOSTNAME" > /etc/hostname - hostname $HOSTNAME - # Delete entry in /etc/hosts derived from dhcp - sed -i '/127.0.1.1/d' /etc/hosts -} - -fix_locale() { - cat >> /etc/default/locale << EOF -LANG=en_US.UTF-8 -LC_ALL=en_US.UTF-8 -EOF - cat >> /etc/locale.gen << EOF -en_US.UTF-8 UTF-8 -EOF - - locale-gen en_US.UTF-8 -} - -# This is actually a bug in the conntrackd package. The comment in the conf file says stats logging is off by default but the parameter is set to on. -# After a couple weeks logrotate will rotate the conntrackd-stats.log file ans start conntracking even if we don't want it to (on non-redundant routers for instance). -fix_conntrackd() { - sed -i '/Stats {/,/}/ s/LogFile on/LogFile off/' /etc/conntrackd/conntrackd.conf - rm -f /var/log/conntrackd-stats.log -} - -fix_vhdutil() { - wget --no-check-certificate http://download.cloud.com.s3.amazonaws.com/tools/vhd-util -O /bin/vhd-util - chmod a+x /bin/vhd-util -} - -# Preload these module otherwise the sysctl settings will not be set, and pasive ftp will not work. -fix_modules() { - cat >> /etc/modules << EOF -nf_conntrack_ipv4 -nf_conntrack -nf_conntrack_ftp -nf_nat_ftp -EOF -} - -do_fixes() { - fix_nameserver - fix_inittab - fix_acpid - fix_hostname - fix_locale - fix_conntrackd - fix_vhdutil - fix_modules -} - -configure_apache2() { - # Enable ssl, rewrite and auth - a2enmod ssl rewrite auth_basic auth_digest - a2ensite default-ssl - # Backup stock apache configuration since we may modify it in Secondary Storage VM - cp /etc/apache2/sites-available/default /etc/apache2/sites-available/default.orig - cp /etc/apache2/sites-available/default-ssl /etc/apache2/sites-available/default-ssl.orig -} - -configure_services() { - local arch=`dpkg --print-architecture` - mkdir -p /var/www/html - mkdir -p /opt/cloud/bin - mkdir -p /var/cache/cloud - mkdir -p /usr/share/cloud - mkdir -p /usr/local/cloud - mkdir -p /root/.ssh - # Fix haproxy directory issue - mkdir -p /var/lib/haproxy - - # Get config files from master - snapshot_url="https://git-wip-us.apache.org/repos/asf?p=cloudstack.git;a=snapshot;h=HEAD;sf=tgz" - snapshot_dir="/opt/cloudstack*" - cd /opt - wget --no-check-certificate $snapshot_url -O cloudstack.tar.gz - tar -zxvf cloudstack.tar.gz --wildcards 'cloudstack-HEAD-???????/systemvm' - cp -rv $snapshot_dir/systemvm/patches/debian/config/* / - cp -rv $snapshot_dir/systemvm/patches/debian/vpn/* / - mkdir -p /usr/share/cloud/ - cd $snapshot_dir/systemvm/patches/debian/config - tar -cvf /usr/share/cloud/cloud-scripts.tar * - cd $snapshot_dir/systemvm/patches/debian/vpn - tar -rvf /usr/share/cloud/cloud-scripts.tar * - cd /opt - rm -fr $snapshot_dir cloudstack.tar.gz - - chkconfig --add cloud-early-config - chkconfig cloud-early-config on - chkconfig --add cloud-passwd-srvr - chkconfig cloud-passwd-srvr off - chkconfig --add cloud - chkconfig cloud off - chkconfig xl2tpd off - # Hyperv kvp daemon - 64bit only - if [ "${arch}" == "amd64" ]; then - chkconfig hv_kvp_daemon off - fi - chkconfig radvd off -} - -do_signature() { - mkdir -p /var/cache/cloud/ - gzip -c /usr/share/cloud/cloud-scripts.tar > /usr/share/cloud/cloud-scripts.tgz - md5sum /usr/share/cloud/cloud-scripts.tgz | awk '{print $1}' > /var/cache/cloud/cloud-scripts-signature - echo "Cloudstack Release $CLOUDSTACK_RELEASE $(date)" > /etc/cloudstack-release -} - -begin=$(date +%s) - -echo "*************ADDING BACKPORTS********************" -add_backports -echo "*************INSTALLING PACKAGES********************" -install_packages -echo "*************DONE INSTALLING PACKAGES********************" -setup_accounts -echo "*************DONE ACCOUNT SETUP********************" -configure_services -configure_apache2 -echo "*************DONE SETTING UP SERVICES********************" -do_fixes -echo "*************DONE FIXING CONFIGURATION********************" -do_signature - -fin=$(date +%s) -t=$((fin-begin)) - -echo "Signed systemvm build, finished building systemvm appliance in $t seconds" diff --git a/tools/appliance/definitions/systemvm64template/preseed.cfg b/tools/appliance/definitions/systemvm64template/preseed.cfg deleted file mode 100644 index 6a10e777de9..00000000000 --- a/tools/appliance/definitions/systemvm64template/preseed.cfg +++ /dev/null @@ -1,360 +0,0 @@ -#### Contents of the preconfiguration file (for squeeze) -### Localization -# Locale sets language and country. -d-i debian-installer/locale string en_US - -# Keyboard selection. -#d-i console-tools/archs select at -d-i console-keymaps-at/keymap select us -# Example for a different keyboard architecture -#d-i console-keymaps-usb/keymap select mac-usb-us - -### Network configuration -# netcfg will choose an interface that has link if possible. This makes it -# skip displaying a list if there is more than one interface. -d-i netcfg/choose_interface select auto - -# To pick a particular interface instead: -#d-i netcfg/choose_interface select eth1 - -# If you have a slow dhcp server and the installer times out waiting for -# it, this might be useful. -#d-i netcfg/dhcp_timeout string 60 - -# If you prefer to configure the network manually, uncomment this line and -# the static network configuration below. -#d-i netcfg/disable_dhcp boolean true - -# If you want the preconfiguration file to work on systems both with and -# without a dhcp server, uncomment these lines and the static network -# configuration below. -#d-i netcfg/dhcp_failed note -#d-i netcfg/dhcp_options select Configure network manually - -# Static network configuration. -#d-i netcfg/get_nameservers string 192.168.1.1 -#d-i netcfg/get_ipaddress string 192.168.1.42 -#d-i netcfg/get_netmask string 255.255.255.0 -#d-i netcfg/get_gateway string 192.168.1.1 -#d-i netcfg/confirm_static boolean true - -# Any hostname and domain names assigned from dhcp take precedence over -# values set here. However, setting the values still prevents the questions -# from being shown, even if values come from dhcp. -d-i netcfg/get_hostname string systemvm -d-i netcfg/get_domain string cloudstack.org - -# Disable that annoying WEP key dialog. -d-i netcfg/wireless_wep string -# The wacky dhcp hostname that some ISPs use as a password of sorts. -#d-i netcfg/dhcp_hostname string radish - -# If non-free firmware is needed for the network or other hardware, you can -# configure the installer to always try to load it, without prompting. Or -# change to false to disable asking. -#d-i hw-detect/load_firmware boolean true - -### Network console -# Use the following settings if you wish to make use of the network-console -# component for remote installation over SSH. This only makes sense if you -# intend to perform the remainder of the installation manually. -#d-i anna/choose_modules string network-console -#d-i network-console/password password r00tme -#d-i network-console/password-again password r00tme - -### Mirror settings -# If you select ftp, the mirror/country string does not need to be set. -#d-i mirror/protocol string ftp -d-i mirror/country string manual -d-i mirror/http/hostname string http.us.debian.org -d-i mirror/http/directory string /debian -d-i mirror/http/proxy string - -# Suite to install. -#d-i mirror/suite string testing -# Suite to use for loading installer components (optional). -#d-i mirror/udeb/suite string testing - -### Clock and time zone setup -# Controls whether or not the hardware clock is set to UTC. -d-i clock-setup/utc boolean true - -# You may set this to any valid setting for $TZ; see the contents of -# /usr/share/zoneinfo/ for valid values. -d-i time/zone string UTC - -# Controls whether to use NTP to set the clock during the install -d-i clock-setup/ntp boolean true -# NTP server to use. The default is almost always fine here. -#d-i clock-setup/ntp-server string ntp.example.com - -### Partitioning -# If the system has free space you can choose to only partition that space. -#d-i partman-auto/init_automatically_partition select biggest_free - -# Alternatively, you can specify a disk to partition. The device name must -# be given in traditional non-devfs format. -# Note: A disk must be specified, unless the system has only one disk. -# For example, to use the first SCSI/SATA hard disk: -d-i partman-auto/disk string /dev/sda -# In addition, you'll need to specify the method to use. -# The presently available methods are: "regular", "lvm" and "crypto" -d-i partman-auto/method string regular - -# If one of the disks that are going to be automatically partitioned -# contains an old LVM configuration, the user will normally receive a -# warning. This can be preseeded away... -#d-i partman-lvm/device_remove_lvm boolean true -# The same applies to pre-existing software RAID array: -#d-i partman-md/device_remove_md boolean true - -# And the same goes for the confirmation to write the lvm partitions. -#d-i partman-lvm/confirm boolean true -#d-i partman-lvm/confirm_nooverwrite boolean true - -#d-i partman/choose_partition select finish -#d-i partman-auto-lvm/guided_size string max - -# You can choose one of the three predefined partitioning recipes: -# - atomic: all files in one partition -# - home: separate /home partition -# - multi: separate /home, /usr, /var, and /tmp partitions -d-i partman-auto/choose_recipe select atomic -#d-i partman/default_filesystem string ext3 - -# Or provide a recipe of your own... -# The recipe format is documented in the file devel/partman-auto-recipe.txt. -# If you have a way to get a recipe file into the d-i environment, you can -# just point at it. -#d-i partman-auto/expert_recipe_file string /hd-media/recipe - -d-i partman-auto/expert_recipe string \ - boot-root :: \ - 30 50 100 ext4 \ - $primary{ } $bootable{ } \ - method{ format } format{ } \ - use_filesystem{ } filesystem{ ext4 } \ - mountpoint{ /boot } \ - . \ - 250 40 400 ext4 \ - method{ format } format{ } \ - use_filesystem{ } filesystem{ ext4 } \ - mountpoint{ / } \ - . \ - 50 100 200 ext4 \ - method{ format } format{ } \ - use_filesystem{ } filesystem{ ext4 } \ - mountpoint{ /home } \ - . \ - 700 70 1100 ext4 \ - method{ format } format{ } \ - use_filesystem{ } filesystem{ ext4 } \ - mountpoint{ /usr } \ - . \ - 400 40 500 ext4 \ - method{ format } format{ } \ - use_filesystem{ } filesystem{ ext4 } \ - mountpoint{ /opt } \ - . \ - 450 70 1000 ext4 \ - method{ format } format{ } \ - use_filesystem{ } filesystem{ ext4 } \ - mountpoint{ /var } \ - . \ - 50 60 100 ext4 \ - method{ format } format{ } \ - use_filesystem{ } filesystem{ ext4 } \ - mountpoint{ /tmp } \ - . \ - 70 100 256 linux-swap \ - method{ swap } format{ } \ - . - -# If not, you can put an entire recipe into the preconfiguration file in one -# (logical) line. This example creates a small /boot partition, suitable -# swap, and uses the rest of the space for the root partition: -#d-i partman-auto/expert_recipe string \ -# boot-root :: \ -# 40 50 100 ext3 \ -# $primary{ } $bootable{ } \ -# method{ format } format{ } \ -# use_filesystem{ } filesystem{ ext3 } \ -# mountpoint{ /boot } \ -# . \ -# 500 10000 1000000000 ext3 \ -# method{ format } format{ } \ -# use_filesystem{ } filesystem{ ext3 } \ -# mountpoint{ / } \ -# . \ -# 64 512 300% linux-swap \ -# method{ swap } format{ } \ -# . - -#The preseed line that "selects finish" needs to be in a certain order in your preseed, the example-preseed does not follow this. -#http://ubuntuforums.org/archive/index.php/t-1504045.html - -# This makes partman automatically partition without confirmation, provided -# that you told it what to do using one of the methods above. -#d-i partman-partitioning/confirm_write_new_label boolean true -d-i partman/confirm_write_new_label boolean true -d-i partman/choose_partition select finish -d-i partman/confirm boolean true -d-i partman/confirm_nooverwrite boolean true - -### Base system installation -# Select the initramfs generator used to generate the initrd for 2.6 kernels. -#d-i base-installer/kernel/linux/initramfs-generators string yaird - -# The kernel image (meta) package to be installed; "none" can be used if no -# kernel is to be installed. -#d-i base-installer/kernel/image string linux-image-2.6-486 - -### Account setup -# Skip creation of a root account (normal user account will be able to -# use sudo). -d-i passwd/root-login boolean true -# Alternatively, to skip creation of a normal user account. -#d-i passwd/make-user boolean false - -# Root password, either in clear text -d-i passwd/root-password password password -d-i passwd/root-password-again password password -# or encrypted using an MD5 hash. -#d-i passwd/root-password-crypted password [MD5 hash] - -# To create a normal user account. -d-i passwd/user-fullname string Cloud Stack -d-i passwd/username string cloud -# Normal user's password, either in clear text -d-i passwd/user-password password cloud -d-i passwd/user-password-again password cloud -# or encrypted using an MD5 hash. -#d-i passwd/user-password-crypted password [MD5 hash] -# Create the first user with the specified UID instead of the default. -#d-i passwd/user-uid string 1010 -d-i user-setup/encrypt-home boolean false -d-i user-setup/allow-password-weak boolean true - -# The user account will be added to some standard initial groups. To -# override that, use this. -d-i passwd/user-default-groups string audio cdrom video admin - -### Apt setup -# You can choose to install non-free and contrib software. -#d-i apt-setup/non-free boolean true -#d-i apt-setup/contrib boolean true -# Uncomment this if you don't want to use a network mirror. -#d-i apt-setup/use_mirror boolean false -# Select which update services to use; define the mirrors to be used. -# Values shown below are the normal defaults. -#d-i apt-setup/services-select multiselect security, volatile -#d-i apt-setup/security_host string security.debian.org -#d-i apt-setup/volatile_host string volatile.debian.org - - -# By default the installer requires that repositories be authenticated -# using a known gpg key. This setting can be used to disable that -# authentication. Warning: Insecure, not recommended. -#d-i debian-installer/allow_unauthenticated string true - -### Package selection -tasksel tasksel/first multiselect ssh-server -# If the desktop task is selected, install the kde and xfce desktops -# instead of the default gnome desktop. -#tasksel tasksel/desktop multiselect kde, xfce - -# Individual additional packages to install -d-i pkgsel/include string openssh-server ntp acpid sudo bzip2 - -# Whether to upgrade packages after debootstrap. -# Allowed values: none, safe-upgrade, full-upgrade -d-i pkgsel/upgrade select none - -# Some versions of the installer can report back on what software you have -# installed, and what software you use. The default is not to report back, -# but sending reports helps the project determine what software is most -# popular and include it on CDs. -popularity-contest popularity-contest/participate boolean false - -### Boot loader installation -# Grub is the default boot loader (for x86). If you want lilo installed -# instead, uncomment this: -#d-i grub-installer/skip boolean true -# To also skip installing lilo, and install no bootloader, uncomment this -# too: -#d-i lilo-installer/skip boolean true - -# This is fairly safe to set, it makes grub install automatically to the MBR -# if no other operating system is detected on the machine. -d-i grub-installer/only_debian boolean true - -# This one makes grub-installer install to the MBR if it also finds some other -# OS, which is less safe as it might not be able to boot that other OS. -#d-i grub-installer/with_other_os boolean true - -# Alternatively, if you want to install to a location other than the mbr, -# uncomment and edit these lines: -#d-i grub-installer/only_debian boolean false -#d-i grub-installer/with_other_os boolean false -#d-i grub-installer/bootdev string (hd0,0) -# To install grub to multiple disks: -#d-i grub-installer/bootdev string (hd0,0) (hd1,0) (hd2,0) - -# Optional password for grub, either in clear text -#d-i grub-installer/password password r00tme -#d-i grub-installer/password-again password r00tme -# or encrypted using an MD5 hash, see grub-md5-crypt(8). -#d-i grub-installer/password-crypted password [MD5 hash] - -### Finishing up the installation -# During installations from serial console, the regular virtual consoles -# (VT1-VT6) are normally disabled in /etc/inittab. Uncomment the next -# line to prevent this. -#d-i finish-install/keep-consoles boolean true - -# Avoid that last message about the install being complete. -d-i finish-install/reboot_in_progress note - -# This will prevent the installer from ejecting the CD during the reboot, -# which is useful in some situations. -#d-i cdrom-detect/eject boolean false - -# This is how to make the installer shutdown when finished, but not -# reboot into the installed system. -#d-i debian-installer/exit/halt boolean true -# This will power off the machine instead of just halting it. -#d-i debian-installer/exit/poweroff boolean true - -### Preseeding other packages -# Depending on what software you choose to install, or if things go wrong -# during the installation process, it's possible that other questions may -# be asked. You can preseed those too, of course. To get a list of every -# possible question that could be asked during an install, do an -# installation, and then run these commands: -# debconf-get-selections --installer > file -# debconf-get-selections >> file - -libssl1.0.0 libssl1.0.0/restart-services string - -libssl1.0.0 libssl1.0.0/restart-failed error - -#### Advanced options -### Running custom commands during the installation -# d-i preseeding is inherently not secure. Nothing in the installer checks -# for attempts at buffer overflows or other exploits of the values of a -# preconfiguration file like this one. Only use preconfiguration files from -# trusted locations! To drive that home, and because it's generally useful, -# here's a way to run any shell command you'd like inside the installer, -# automatically. - -# This first command is run as early as possible, just after -# preseeding is read. -# Prevent packaged version of VirtualBox Guest Additions being installed: -d-i preseed/early_command string sed -i \ - '/in-target/idiscover(){/sbin/discover|grep -v VirtualBox;}' \ - /usr/lib/pre-pkgsel.d/20install-hwpackages - -# This command is run just before the install finishes, but when there is -# still a usable /target directory. You can chroot to /target and use it -# directly, or use the apt-install and in-target commands to easily install -# packages and run commands in the target system. diff --git a/tools/appliance/definitions/systemvm64template/zerodisk.sh b/tools/appliance/definitions/systemvm64template/zerodisk.sh deleted file mode 100644 index b00f7ae7ccc..00000000000 --- a/tools/appliance/definitions/systemvm64template/zerodisk.sh +++ /dev/null @@ -1,15 +0,0 @@ -# Clean up stuff copied in by veewee -rm -fv /root/*.iso -rm -fv /root/base.sh /root/cleanup.sh /root/postinstall.sh /root/zerodisk.sh -rm -fv .veewee_version .veewee_params .vbox_version - -echo "Cleaning up" - -# Zero out the free space to save space in the final image: -for path in / /boot /usr /var /opt /tmp /home -do - dd if=/dev/zero of=$path/zero bs=1M - sync - rm -f $path/zero - echo "Completed zero-ing out disk on $path" -done From 2210f1b1da742ddcf5a5a36196d4a8dc9935f214 Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 10:52:01 +0200 Subject: [PATCH 06/60] CLOUDSTACK-7143: remove 'manual' from preseed config. No other changes. --- .../definitions/systemvmtemplate/preseed.cfg | 256 +----------------- 1 file changed, 4 insertions(+), 252 deletions(-) diff --git a/tools/appliance/definitions/systemvmtemplate/preseed.cfg b/tools/appliance/definitions/systemvmtemplate/preseed.cfg index 6a10e777de9..293d6534547 100644 --- a/tools/appliance/definitions/systemvmtemplate/preseed.cfg +++ b/tools/appliance/definitions/systemvmtemplate/preseed.cfg @@ -1,133 +1,30 @@ -#### Contents of the preconfiguration file (for squeeze) ### Localization # Locale sets language and country. d-i debian-installer/locale string en_US # Keyboard selection. -#d-i console-tools/archs select at d-i console-keymaps-at/keymap select us -# Example for a different keyboard architecture -#d-i console-keymaps-usb/keymap select mac-usb-us ### Network configuration -# netcfg will choose an interface that has link if possible. This makes it -# skip displaying a list if there is more than one interface. d-i netcfg/choose_interface select auto - -# To pick a particular interface instead: -#d-i netcfg/choose_interface select eth1 - -# If you have a slow dhcp server and the installer times out waiting for -# it, this might be useful. -#d-i netcfg/dhcp_timeout string 60 - -# If you prefer to configure the network manually, uncomment this line and -# the static network configuration below. -#d-i netcfg/disable_dhcp boolean true - -# If you want the preconfiguration file to work on systems both with and -# without a dhcp server, uncomment these lines and the static network -# configuration below. -#d-i netcfg/dhcp_failed note -#d-i netcfg/dhcp_options select Configure network manually - -# Static network configuration. -#d-i netcfg/get_nameservers string 192.168.1.1 -#d-i netcfg/get_ipaddress string 192.168.1.42 -#d-i netcfg/get_netmask string 255.255.255.0 -#d-i netcfg/get_gateway string 192.168.1.1 -#d-i netcfg/confirm_static boolean true - -# Any hostname and domain names assigned from dhcp take precedence over -# values set here. However, setting the values still prevents the questions -# from being shown, even if values come from dhcp. d-i netcfg/get_hostname string systemvm d-i netcfg/get_domain string cloudstack.org -# Disable that annoying WEP key dialog. -d-i netcfg/wireless_wep string -# The wacky dhcp hostname that some ISPs use as a password of sorts. -#d-i netcfg/dhcp_hostname string radish - -# If non-free firmware is needed for the network or other hardware, you can -# configure the installer to always try to load it, without prompting. Or -# change to false to disable asking. -#d-i hw-detect/load_firmware boolean true - -### Network console -# Use the following settings if you wish to make use of the network-console -# component for remote installation over SSH. This only makes sense if you -# intend to perform the remainder of the installation manually. -#d-i anna/choose_modules string network-console -#d-i network-console/password password r00tme -#d-i network-console/password-again password r00tme - ### Mirror settings -# If you select ftp, the mirror/country string does not need to be set. -#d-i mirror/protocol string ftp d-i mirror/country string manual d-i mirror/http/hostname string http.us.debian.org d-i mirror/http/directory string /debian d-i mirror/http/proxy string -# Suite to install. -#d-i mirror/suite string testing -# Suite to use for loading installer components (optional). -#d-i mirror/udeb/suite string testing - ### Clock and time zone setup -# Controls whether or not the hardware clock is set to UTC. d-i clock-setup/utc boolean true - -# You may set this to any valid setting for $TZ; see the contents of -# /usr/share/zoneinfo/ for valid values. d-i time/zone string UTC - -# Controls whether to use NTP to set the clock during the install d-i clock-setup/ntp boolean true -# NTP server to use. The default is almost always fine here. -#d-i clock-setup/ntp-server string ntp.example.com ### Partitioning -# If the system has free space you can choose to only partition that space. -#d-i partman-auto/init_automatically_partition select biggest_free - -# Alternatively, you can specify a disk to partition. The device name must -# be given in traditional non-devfs format. -# Note: A disk must be specified, unless the system has only one disk. -# For example, to use the first SCSI/SATA hard disk: d-i partman-auto/disk string /dev/sda -# In addition, you'll need to specify the method to use. -# The presently available methods are: "regular", "lvm" and "crypto" d-i partman-auto/method string regular - -# If one of the disks that are going to be automatically partitioned -# contains an old LVM configuration, the user will normally receive a -# warning. This can be preseeded away... -#d-i partman-lvm/device_remove_lvm boolean true -# The same applies to pre-existing software RAID array: -#d-i partman-md/device_remove_md boolean true - -# And the same goes for the confirmation to write the lvm partitions. -#d-i partman-lvm/confirm boolean true -#d-i partman-lvm/confirm_nooverwrite boolean true - -#d-i partman/choose_partition select finish -#d-i partman-auto-lvm/guided_size string max - -# You can choose one of the three predefined partitioning recipes: -# - atomic: all files in one partition -# - home: separate /home partition -# - multi: separate /home, /usr, /var, and /tmp partitions d-i partman-auto/choose_recipe select atomic -#d-i partman/default_filesystem string ext3 - -# Or provide a recipe of your own... -# The recipe format is documented in the file devel/partman-auto-recipe.txt. -# If you have a way to get a recipe file into the d-i environment, you can -# just point at it. -#d-i partman-auto/expert_recipe_file string /hd-media/recipe - d-i partman-auto/expert_recipe string \ boot-root :: \ 30 50 100 ext4 \ @@ -169,192 +66,47 @@ d-i partman-auto/expert_recipe string \ 70 100 256 linux-swap \ method{ swap } format{ } \ . - -# If not, you can put an entire recipe into the preconfiguration file in one -# (logical) line. This example creates a small /boot partition, suitable -# swap, and uses the rest of the space for the root partition: -#d-i partman-auto/expert_recipe string \ -# boot-root :: \ -# 40 50 100 ext3 \ -# $primary{ } $bootable{ } \ -# method{ format } format{ } \ -# use_filesystem{ } filesystem{ ext3 } \ -# mountpoint{ /boot } \ -# . \ -# 500 10000 1000000000 ext3 \ -# method{ format } format{ } \ -# use_filesystem{ } filesystem{ ext3 } \ -# mountpoint{ / } \ -# . \ -# 64 512 300% linux-swap \ -# method{ swap } format{ } \ -# . - -#The preseed line that "selects finish" needs to be in a certain order in your preseed, the example-preseed does not follow this. -#http://ubuntuforums.org/archive/index.php/t-1504045.html - -# This makes partman automatically partition without confirmation, provided -# that you told it what to do using one of the methods above. -#d-i partman-partitioning/confirm_write_new_label boolean true d-i partman/confirm_write_new_label boolean true d-i partman/choose_partition select finish d-i partman/confirm boolean true d-i partman/confirm_nooverwrite boolean true ### Base system installation -# Select the initramfs generator used to generate the initrd for 2.6 kernels. -#d-i base-installer/kernel/linux/initramfs-generators string yaird - -# The kernel image (meta) package to be installed; "none" can be used if no -# kernel is to be installed. -#d-i base-installer/kernel/image string linux-image-2.6-486 +# ... ### Account setup -# Skip creation of a root account (normal user account will be able to -# use sudo). d-i passwd/root-login boolean true -# Alternatively, to skip creation of a normal user account. -#d-i passwd/make-user boolean false - -# Root password, either in clear text d-i passwd/root-password password password d-i passwd/root-password-again password password -# or encrypted using an MD5 hash. -#d-i passwd/root-password-crypted password [MD5 hash] - -# To create a normal user account. d-i passwd/user-fullname string Cloud Stack d-i passwd/username string cloud -# Normal user's password, either in clear text d-i passwd/user-password password cloud d-i passwd/user-password-again password cloud -# or encrypted using an MD5 hash. -#d-i passwd/user-password-crypted password [MD5 hash] -# Create the first user with the specified UID instead of the default. -#d-i passwd/user-uid string 1010 d-i user-setup/encrypt-home boolean false d-i user-setup/allow-password-weak boolean true - -# The user account will be added to some standard initial groups. To -# override that, use this. d-i passwd/user-default-groups string audio cdrom video admin ### Apt setup -# You can choose to install non-free and contrib software. -#d-i apt-setup/non-free boolean true -#d-i apt-setup/contrib boolean true -# Uncomment this if you don't want to use a network mirror. -#d-i apt-setup/use_mirror boolean false -# Select which update services to use; define the mirrors to be used. -# Values shown below are the normal defaults. -#d-i apt-setup/services-select multiselect security, volatile -#d-i apt-setup/security_host string security.debian.org -#d-i apt-setup/volatile_host string volatile.debian.org - - -# By default the installer requires that repositories be authenticated -# using a known gpg key. This setting can be used to disable that -# authentication. Warning: Insecure, not recommended. -#d-i debian-installer/allow_unauthenticated string true +# ... ### Package selection tasksel tasksel/first multiselect ssh-server -# If the desktop task is selected, install the kde and xfce desktops -# instead of the default gnome desktop. -#tasksel tasksel/desktop multiselect kde, xfce - -# Individual additional packages to install d-i pkgsel/include string openssh-server ntp acpid sudo bzip2 - -# Whether to upgrade packages after debootstrap. # Allowed values: none, safe-upgrade, full-upgrade d-i pkgsel/upgrade select none -# Some versions of the installer can report back on what software you have -# installed, and what software you use. The default is not to report back, -# but sending reports helps the project determine what software is most -# popular and include it on CDs. popularity-contest popularity-contest/participate boolean false ### Boot loader installation -# Grub is the default boot loader (for x86). If you want lilo installed -# instead, uncomment this: -#d-i grub-installer/skip boolean true -# To also skip installing lilo, and install no bootloader, uncomment this -# too: -#d-i lilo-installer/skip boolean true - -# This is fairly safe to set, it makes grub install automatically to the MBR -# if no other operating system is detected on the machine. d-i grub-installer/only_debian boolean true - -# This one makes grub-installer install to the MBR if it also finds some other -# OS, which is less safe as it might not be able to boot that other OS. -#d-i grub-installer/with_other_os boolean true - -# Alternatively, if you want to install to a location other than the mbr, -# uncomment and edit these lines: -#d-i grub-installer/only_debian boolean false -#d-i grub-installer/with_other_os boolean false -#d-i grub-installer/bootdev string (hd0,0) -# To install grub to multiple disks: -#d-i grub-installer/bootdev string (hd0,0) (hd1,0) (hd2,0) - -# Optional password for grub, either in clear text -#d-i grub-installer/password password r00tme -#d-i grub-installer/password-again password r00tme -# or encrypted using an MD5 hash, see grub-md5-crypt(8). -#d-i grub-installer/password-crypted password [MD5 hash] - -### Finishing up the installation -# During installations from serial console, the regular virtual consoles -# (VT1-VT6) are normally disabled in /etc/inittab. Uncomment the next -# line to prevent this. -#d-i finish-install/keep-consoles boolean true - -# Avoid that last message about the install being complete. d-i finish-install/reboot_in_progress note -# This will prevent the installer from ejecting the CD during the reboot, -# which is useful in some situations. -#d-i cdrom-detect/eject boolean false - -# This is how to make the installer shutdown when finished, but not -# reboot into the installed system. -#d-i debian-installer/exit/halt boolean true -# This will power off the machine instead of just halting it. -#d-i debian-installer/exit/poweroff boolean true - ### Preseeding other packages -# Depending on what software you choose to install, or if things go wrong -# during the installation process, it's possible that other questions may -# be asked. You can preseed those too, of course. To get a list of every -# possible question that could be asked during an install, do an -# installation, and then run these commands: -# debconf-get-selections --installer > file -# debconf-get-selections >> file - -libssl1.0.0 libssl1.0.0/restart-services string - -libssl1.0.0 libssl1.0.0/restart-failed error +libssl1.0.0 libssl1.0.0/restart-services string +libssl1.0.0 libssl1.0.0/restart-failed error #### Advanced options -### Running custom commands during the installation -# d-i preseeding is inherently not secure. Nothing in the installer checks -# for attempts at buffer overflows or other exploits of the values of a -# preconfiguration file like this one. Only use preconfiguration files from -# trusted locations! To drive that home, and because it's generally useful, -# here's a way to run any shell command you'd like inside the installer, -# automatically. - -# This first command is run as early as possible, just after -# preseeding is read. # Prevent packaged version of VirtualBox Guest Additions being installed: d-i preseed/early_command string sed -i \ '/in-target/idiscover(){/sbin/discover|grep -v VirtualBox;}' \ /usr/lib/pre-pkgsel.d/20install-hwpackages - -# This command is run just before the install finishes, but when there is -# still a usable /target directory. You can chroot to /target and use it -# directly, or use the apt-install and in-target commands to easily install -# packages and run commands in the target system. From 5f3caf6d01dc3f9dd70fbe31083a46c56834659d Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 10:54:57 +0200 Subject: [PATCH 07/60] CLOUDSTACK-7143: reformat definition.rb code. No other changes. --- .../systemvmtemplate/definition.rb | 47 ++++++++++--------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/tools/appliance/definitions/systemvmtemplate/definition.rb b/tools/appliance/definitions/systemvmtemplate/definition.rb index c57bf237b33..0a58746d247 100644 --- a/tools/appliance/definitions/systemvmtemplate/definition.rb +++ b/tools/appliance/definitions/systemvmtemplate/definition.rb @@ -10,9 +10,9 @@ arch = ENV['VM_ARCH'] || 'i386' architectures = { :i386 => { :os_type_id => 'Debian', - :iso_file => "debian-7.6.0-i386-netinst.iso", - :iso_src => "http://cdimage.debian.org/debian-cd/7.6.0/i386/iso-cd/debian-7.6.0-i386-netinst.iso", - :iso_md5 => "528e1a7315da1bbf50bd4d187880a519", + :iso_file => 'debian-7.6.0-i386-netinst.iso', + :iso_src => 'http://cdimage.debian.org/debian-cd/7.6.0/i386/iso-cd/debian-7.6.0-i386-netinst.iso', + :iso_md5 => '528e1a7315da1bbf50bd4d187880a519', }, :amd64 => { :os_type_id => 'Debian_64', @@ -24,10 +24,11 @@ architectures = { config = { :cpu_count => '1', - :memory_size=> '256', + :memory_size => '256', :disk_size => '2500', :disk_format => 'VDI', :hostiocache => 'off', - :iso_download_timeout => "1000", - :boot_wait => "10", :boot_cmd_sequence => [ + :iso_download_timeout => '1000', + :boot_wait => '10', + :boot_cmd_sequence => [ '', 'install ', 'preseed/url=http://%IP%:%PORT%/preseed.cfg ', @@ -44,26 +45,26 @@ config = { 'keyboard-configuration/xkb-keymap=us ', '' ], - :kickstart_port => "7122", - :kickstart_timeout => "10000", - :kickstart_file => "preseed.cfg", - :ssh_login_timeout => "10000", - :ssh_user => "root", - :ssh_password => "password", - :ssh_key => "", - :ssh_host_port => "7222", - :ssh_guest_port => "22", + :kickstart_port => '7122', + :kickstart_timeout => '10000', + :kickstart_file => 'preseed.cfg', + :ssh_login_timeout => '10000', + :ssh_user => 'root', + :ssh_password => 'password', + :ssh_key => '', + :ssh_host_port => '7222', + :ssh_guest_port => '22', :sudo_cmd => "echo '%p'|sudo -S sh '%f'", - :shutdown_cmd => "halt -p", + :shutdown_cmd => 'halt -p', :postinstall_files => [ - "apt_upgrade.sh", - "configure_grub.sh", - "postinstall.sh", - "cleanup.sh", - "configure_networking.sh", - "zerodisk.sh" + 'apt_upgrade.sh', + 'configure_grub.sh', + 'postinstall.sh', + 'cleanup.sh', + 'configure_networking.sh', + 'zerodisk.sh' ], - :postinstall_timeout => "10000" + :postinstall_timeout => '10000' } config.merge! architectures[arch.to_sym] From 6641fd3a76eb59cbc8cf73a645351e0fe6211289 Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 10:57:32 +0200 Subject: [PATCH 08/60] CLOUDSTACK-7143: move add_backports to apt_upgrade.sh --- .../appliance/definitions/systemvmtemplate/apt_upgrade.sh | 7 +++++++ .../appliance/definitions/systemvmtemplate/postinstall.sh | 8 -------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/tools/appliance/definitions/systemvmtemplate/apt_upgrade.sh b/tools/appliance/definitions/systemvmtemplate/apt_upgrade.sh index 69acc1a9c6b..6d38a24a739 100644 --- a/tools/appliance/definitions/systemvmtemplate/apt_upgrade.sh +++ b/tools/appliance/definitions/systemvmtemplate/apt_upgrade.sh @@ -3,6 +3,13 @@ export DEBIAN_FRONTEND=noninteractive export DEBIAN_PRIORITY=critical +add_backports () { + sed -i '/backports/d' /etc/apt/sources.list + echo 'deb http://http.us.debian.org/debian wheezy-backports main' >> /etc/apt/sources.list +} + +add_backports + apt-get -q -y --force-yes update apt-get -q -y --force-yes install curl unzip apt-get clean diff --git a/tools/appliance/definitions/systemvmtemplate/postinstall.sh b/tools/appliance/definitions/systemvmtemplate/postinstall.sh index e00d259ec8a..1cf60869ff9 100644 --- a/tools/appliance/definitions/systemvmtemplate/postinstall.sh +++ b/tools/appliance/definitions/systemvmtemplate/postinstall.sh @@ -21,12 +21,6 @@ ROOTPW=password HOSTNAME=systemvm CLOUDSTACK_RELEASE=4.5.0 -add_backports () { - sed -i '/backports/d' /etc/apt/sources.list - echo 'deb http://http.us.debian.org/debian wheezy-backports main' >> /etc/apt/sources.list - apt-get update -} - install_packages() { DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical @@ -280,8 +274,6 @@ do_signature() { begin=$(date +%s) -echo "*************ADDING BACKPORTS********************" -add_backports echo "*************INSTALLING PACKAGES********************" install_packages echo "*************DONE INSTALLING PACKAGES********************" From 58ca92ce527e7cee50f67508d49d3aaa6c32a348 Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 10:58:57 +0200 Subject: [PATCH 09/60] CLOUDSTACK-7143: cleanup apt_upgrade.sh code --- .../systemvmtemplate/apt_upgrade.sh | 26 ++++++++++++------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/tools/appliance/definitions/systemvmtemplate/apt_upgrade.sh b/tools/appliance/definitions/systemvmtemplate/apt_upgrade.sh index 6d38a24a739..640d572ffe1 100644 --- a/tools/appliance/definitions/systemvmtemplate/apt_upgrade.sh +++ b/tools/appliance/definitions/systemvmtemplate/apt_upgrade.sh @@ -1,15 +1,21 @@ -# Update the box +#!/bin/bash -export DEBIAN_FRONTEND=noninteractive -export DEBIAN_PRIORITY=critical +set -e +set -x -add_backports () { - sed -i '/backports/d' /etc/apt/sources.list - echo 'deb http://http.us.debian.org/debian wheezy-backports main' >> /etc/apt/sources.list +function add_backports() { + sed -i '/backports/d' /etc/apt/sources.list + echo 'deb http://http.us.debian.org/debian wheezy-backports main' >> /etc/apt/sources.list } -add_backports +function apt_upgrade() { + DEBIAN_FRONTEND=noninteractive + DEBIAN_PRIORITY=critical -apt-get -q -y --force-yes update -apt-get -q -y --force-yes install curl unzip -apt-get clean + add_backports + + apt-get -q -y --force-yes update + apt-get -q -y --force-yes upgrade +} + +return 2>/dev/null || apt_upgrade From 29c1f91aea635edbf17d74462f8fda112355153e Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 11:01:15 +0200 Subject: [PATCH 10/60] CLOUDSTACK-7143: add vagrant_box_build_time file that's standard with current veewee templates --- .../definitions/systemvmtemplate/build_time.sh | 10 ++++++++++ .../definitions/systemvmtemplate/definition.rb | 1 + 2 files changed, 11 insertions(+) create mode 100644 tools/appliance/definitions/systemvmtemplate/build_time.sh diff --git a/tools/appliance/definitions/systemvmtemplate/build_time.sh b/tools/appliance/definitions/systemvmtemplate/build_time.sh new file mode 100644 index 00000000000..d44b9dbb155 --- /dev/null +++ b/tools/appliance/definitions/systemvmtemplate/build_time.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +set -e +set -x + +function build_time() { + date > /etc/vagrant_box_build_time +} + +return 2>/dev/null || build_time diff --git a/tools/appliance/definitions/systemvmtemplate/definition.rb b/tools/appliance/definitions/systemvmtemplate/definition.rb index 0a58746d247..b207b740b8e 100644 --- a/tools/appliance/definitions/systemvmtemplate/definition.rb +++ b/tools/appliance/definitions/systemvmtemplate/definition.rb @@ -57,6 +57,7 @@ config = { :sudo_cmd => "echo '%p'|sudo -S sh '%f'", :shutdown_cmd => 'halt -p', :postinstall_files => [ + 'build_time.sh', 'apt_upgrade.sh', 'configure_grub.sh', 'postinstall.sh', From e5a2e67551551974ff639873d78cebe64e7bab5e Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 11:01:56 +0200 Subject: [PATCH 11/60] CLOUDSTACK-7143: cleanup configure_gruh.sh code Adding a check whether GRUB_TIMEOUT has already been set, and if it has, skip re-running update-grub. --- .../systemvmtemplate/configure_grub.sh | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/tools/appliance/definitions/systemvmtemplate/configure_grub.sh b/tools/appliance/definitions/systemvmtemplate/configure_grub.sh index d530bb93bf9..7978bec5301 100644 --- a/tools/appliance/definitions/systemvmtemplate/configure_grub.sh +++ b/tools/appliance/definitions/systemvmtemplate/configure_grub.sh @@ -1,5 +1,13 @@ +#!/bin/bash + +set -e +set -x + # Remove 5s grub timeout to speed up booting -cat < /etc/default/grub +function configure_grub() { + grep GRUB_TIMEOUT=0 /etc/default/grub && return + + cat < /etc/default/grub # If you change this file, run 'update-grub' afterwards to update # /boot/grub/grub.cfg. @@ -10,4 +18,7 @@ GRUB_CMDLINE_LINUX_DEFAULT="quiet" GRUB_CMDLINE_LINUX="debian-installer=en_US" EOF -update-grub + update-grub +} + +return 2>/dev/null || configure_grub From e86121db7b07c67590151f239627d45366c1a095 Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 11:09:41 +0200 Subject: [PATCH 12/60] CLOUDSTACK-7143: split login config out from postinstall.sh --- .../systemvmtemplate/configure_login.sh | 32 ++++++++++++++++++ .../systemvmtemplate/definition.rb | 1 + .../systemvmtemplate/postinstall.sh | 33 ------------------- 3 files changed, 33 insertions(+), 33 deletions(-) create mode 100644 tools/appliance/definitions/systemvmtemplate/configure_login.sh diff --git a/tools/appliance/definitions/systemvmtemplate/configure_login.sh b/tools/appliance/definitions/systemvmtemplate/configure_login.sh new file mode 100644 index 00000000000..413d4852a65 --- /dev/null +++ b/tools/appliance/definitions/systemvmtemplate/configure_login.sh @@ -0,0 +1,32 @@ +setup_accounts() { + # Setup sudo to allow no-password sudo for "admin" + groupadd -r admin + # Create a 'cloud' user if it's not there + id cloud + if [[ $? -ne 0 ]] + then + useradd -G admin cloud + else + usermod -a -G admin cloud + fi + echo "root:$ROOTPW" | chpasswd + echo "cloud:`openssl rand -base64 32`" | chpasswd + sed -i -e '/Defaults\s\+env_reset/a Defaults\texempt_group=admin' /etc/sudoers + sed -i -e 's/%admin ALL=(ALL) ALL/%admin ALL=NOPASSWD:/bin/chmod, /bin/cp, /bin/mkdir, /bin/mount, /bin/umount/g' /etc/sudoers + # Disable password based authentication via ssh, this will take effect on next reboot + sed -i -e 's/^.*PasswordAuthentication .*$/PasswordAuthentication no/g' /etc/ssh/sshd_config + # Secure ~/.ssh + mkdir -p /home/cloud/.ssh + chmod 700 /home/cloud/.ssh +} + +fix_inittab() { + # Fix inittab + cat >> /etc/inittab << EOF + +vc:2345:respawn:/sbin/getty 38400 hvc0 +EOF +} + +setup_accounts +fix_inittab diff --git a/tools/appliance/definitions/systemvmtemplate/definition.rb b/tools/appliance/definitions/systemvmtemplate/definition.rb index b207b740b8e..3513d4678a9 100644 --- a/tools/appliance/definitions/systemvmtemplate/definition.rb +++ b/tools/appliance/definitions/systemvmtemplate/definition.rb @@ -60,6 +60,7 @@ config = { 'build_time.sh', 'apt_upgrade.sh', 'configure_grub.sh', + 'configure_login.sh', 'postinstall.sh', 'cleanup.sh', 'configure_networking.sh', diff --git a/tools/appliance/definitions/systemvmtemplate/postinstall.sh b/tools/appliance/definitions/systemvmtemplate/postinstall.sh index 1cf60869ff9..aaa023e1850 100644 --- a/tools/appliance/definitions/systemvmtemplate/postinstall.sh +++ b/tools/appliance/definitions/systemvmtemplate/postinstall.sh @@ -108,28 +108,6 @@ install_packages() { apt-get --no-install-recommends -q -y --force-yes install radvd } -setup_accounts() { - # Setup sudo to allow no-password sudo for "admin" - groupadd -r admin - # Create a 'cloud' user if it's not there - id cloud - if [[ $? -ne 0 ]] - then - useradd -G admin cloud - else - usermod -a -G admin cloud - fi - echo "root:$ROOTPW" | chpasswd - echo "cloud:`openssl rand -base64 32`" | chpasswd - sed -i -e '/Defaults\s\+env_reset/a Defaults\texempt_group=admin' /etc/sudoers - sed -i -e 's/%admin ALL=(ALL) ALL/%admin ALL=NOPASSWD:/bin/chmod, /bin/cp, /bin/mkdir, /bin/mount, /bin/umount/g' /etc/sudoers - # Disable password based authentication via ssh, this will take effect on next reboot - sed -i -e 's/^.*PasswordAuthentication .*$/PasswordAuthentication no/g' /etc/ssh/sshd_config - # Secure ~/.ssh - mkdir -p /home/cloud/.ssh - chmod 700 /home/cloud/.ssh -} - fix_nameserver() { # Replace /etc/resolv.conf also cat > /etc/resolv.conf << EOF @@ -138,14 +116,6 @@ nameserver 8.8.4.4 EOF } -fix_inittab() { - # Fix inittab - cat >> /etc/inittab << EOF - -vc:2345:respawn:/sbin/getty 38400 hvc0 -EOF -} - fix_acpid() { # Fix acpid mkdir -p /etc/acpi/events @@ -206,7 +176,6 @@ EOF do_fixes() { fix_nameserver - fix_inittab fix_acpid fix_hostname fix_locale @@ -277,8 +246,6 @@ begin=$(date +%s) echo "*************INSTALLING PACKAGES********************" install_packages echo "*************DONE INSTALLING PACKAGES********************" -setup_accounts -echo "*************DONE ACCOUNT SETUP********************" configure_services configure_apache2 echo "*************DONE SETTING UP SERVICES********************" From 3f8c31b0da89e630b27b26d495ad1babbf57678a Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 11:10:59 +0200 Subject: [PATCH 13/60] CLOUDSTACK-7143: cleanup configure_login.sh code Had to change various things to make this code re-entrant. In particular, the sed-based manipulation of /etc/sudoers is gone and replaced with a simpler, minimal (but compatible) sudoers file. Remove the sshd_config tuning since sshd_config is overwritten when we apply the cloud_scripts overlay (from build.sh). --- .../systemvmtemplate/configure_login.sh | 76 +++++++++++++------ 1 file changed, 53 insertions(+), 23 deletions(-) diff --git a/tools/appliance/definitions/systemvmtemplate/configure_login.sh b/tools/appliance/definitions/systemvmtemplate/configure_login.sh index 413d4852a65..680b08a7818 100644 --- a/tools/appliance/definitions/systemvmtemplate/configure_login.sh +++ b/tools/appliance/definitions/systemvmtemplate/configure_login.sh @@ -1,26 +1,49 @@ -setup_accounts() { - # Setup sudo to allow no-password sudo for "admin" - groupadd -r admin - # Create a 'cloud' user if it's not there - id cloud - if [[ $? -ne 0 ]] - then - useradd -G admin cloud - else - usermod -a -G admin cloud - fi - echo "root:$ROOTPW" | chpasswd - echo "cloud:`openssl rand -base64 32`" | chpasswd - sed -i -e '/Defaults\s\+env_reset/a Defaults\texempt_group=admin' /etc/sudoers - sed -i -e 's/%admin ALL=(ALL) ALL/%admin ALL=NOPASSWD:/bin/chmod, /bin/cp, /bin/mkdir, /bin/mount, /bin/umount/g' /etc/sudoers - # Disable password based authentication via ssh, this will take effect on next reboot - sed -i -e 's/^.*PasswordAuthentication .*$/PasswordAuthentication no/g' /etc/ssh/sshd_config - # Secure ~/.ssh - mkdir -p /home/cloud/.ssh - chmod 700 /home/cloud/.ssh +#!/bin/bash + +set -e +set -x + +function add_admin_group() { + groupadd -f -r admin } -fix_inittab() { +function configure_cloud_user() { + usermod -a -G admin cloud + mkdir -p /home/cloud/.ssh + chmod 700 /home/cloud/.ssh + echo "cloud:`openssl rand -base64 32`" | chpasswd +} + +function configure_sudoers() { + cat >/etc/sudoers < /etc/sudoers.d/cloud +} + +# sshd_config is overwritten from cloud_scripts +#function configure_sshd() { +# grep "UseDNS no" /etc/ssh/sshd_config && \ +# grep "PasswordAuthentication no" /etc/ssh/sshd_config && \ +# return +# # Tweak sshd to prevent DNS resolution (speed up logins) +# echo 'UseDNS no' >> /etc/ssh/sshd_config +# +# # Require ssh keys for login +# sed -i -e 's/^.*PasswordAuthentication .*$/PasswordAuthentication no/g' /etc/ssh/sshd_config +#} + +function configure_inittab() { + grep "vc:2345:respawn:/sbin/getty" /etc/inittab && return + # Fix inittab cat >> /etc/inittab << EOF @@ -28,5 +51,12 @@ vc:2345:respawn:/sbin/getty 38400 hvc0 EOF } -setup_accounts -fix_inittab +function configure_login() { + add_admin_group + configure_cloud_user + configure_sudoers + # configure_sshd + configure_inittab +} + +return 2>/dev/null || configure_login From aa14a619a640e21c6f3be18c6a0500e6b7082b4f Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 11:15:23 +0200 Subject: [PATCH 14/60] CLOUDSTACK-7143: split locale config into its own file --- .../systemvmtemplate/configure_locale.sh | 13 +++++++++++++ .../definitions/systemvmtemplate/definition.rb | 1 + .../definitions/systemvmtemplate/postinstall.sh | 13 ------------- 3 files changed, 14 insertions(+), 13 deletions(-) create mode 100644 tools/appliance/definitions/systemvmtemplate/configure_locale.sh diff --git a/tools/appliance/definitions/systemvmtemplate/configure_locale.sh b/tools/appliance/definitions/systemvmtemplate/configure_locale.sh new file mode 100644 index 00000000000..6619c6cf11b --- /dev/null +++ b/tools/appliance/definitions/systemvmtemplate/configure_locale.sh @@ -0,0 +1,13 @@ +fix_locale() { + cat >> /etc/default/locale << EOF +LANG=en_US.UTF-8 +LC_ALL=en_US.UTF-8 +EOF + cat >> /etc/locale.gen << EOF +en_US.UTF-8 UTF-8 +EOF + + locale-gen en_US.UTF-8 +} + +fix_locale diff --git a/tools/appliance/definitions/systemvmtemplate/definition.rb b/tools/appliance/definitions/systemvmtemplate/definition.rb index 3513d4678a9..be0b4036ce0 100644 --- a/tools/appliance/definitions/systemvmtemplate/definition.rb +++ b/tools/appliance/definitions/systemvmtemplate/definition.rb @@ -60,6 +60,7 @@ config = { 'build_time.sh', 'apt_upgrade.sh', 'configure_grub.sh', + 'configure_locale.sh', 'configure_login.sh', 'postinstall.sh', 'cleanup.sh', diff --git a/tools/appliance/definitions/systemvmtemplate/postinstall.sh b/tools/appliance/definitions/systemvmtemplate/postinstall.sh index aaa023e1850..893b5214fb1 100644 --- a/tools/appliance/definitions/systemvmtemplate/postinstall.sh +++ b/tools/appliance/definitions/systemvmtemplate/postinstall.sh @@ -140,18 +140,6 @@ fix_hostname() { sed -i '/127.0.1.1/d' /etc/hosts } -fix_locale() { - cat >> /etc/default/locale << EOF -LANG=en_US.UTF-8 -LC_ALL=en_US.UTF-8 -EOF - cat >> /etc/locale.gen << EOF -en_US.UTF-8 UTF-8 -EOF - - locale-gen en_US.UTF-8 -} - # This is actually a bug in the conntrackd package. The comment in the conf file says stats logging is off by default but the parameter is set to on. # After a couple weeks logrotate will rotate the conntrackd-stats.log file ans start conntracking even if we don't want it to (on non-redundant routers for instance). fix_conntrackd() { @@ -178,7 +166,6 @@ do_fixes() { fix_nameserver fix_acpid fix_hostname - fix_locale fix_conntrackd fix_vhdutil fix_modules From 6a688a0337c4faa6b96552dccd620b3b58d00a4c Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 11:16:48 +0200 Subject: [PATCH 15/60] CLOUDSTACK-7143: cleanup configure_locale.sh code --- .../systemvmtemplate/configure_locale.sh | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tools/appliance/definitions/systemvmtemplate/configure_locale.sh b/tools/appliance/definitions/systemvmtemplate/configure_locale.sh index 6619c6cf11b..8747a571d34 100644 --- a/tools/appliance/definitions/systemvmtemplate/configure_locale.sh +++ b/tools/appliance/definitions/systemvmtemplate/configure_locale.sh @@ -1,4 +1,14 @@ -fix_locale() { +#!/bin/bash + +set -e +set -x + +function configure_locale() { + grep LANG=en_US.UTF-8 /etc/default/locale && \ + grep LC_ALL=en_US.UTF-8 /etc/default/locale && \ + grep "en_US.UTF-8 UTF-8" /etc/locale.gen && + return + cat >> /etc/default/locale << EOF LANG=en_US.UTF-8 LC_ALL=en_US.UTF-8 @@ -10,4 +20,4 @@ EOF locale-gen en_US.UTF-8 } -fix_locale +return 2>/dev/null || configure_locale From 5627b67ff3a6af70949ee1622b3e5a572d39a0b7 Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 11:19:03 +0200 Subject: [PATCH 16/60] CLOUDSTACK-7143: move fix_acpid to its own file --- .../systemvmtemplate/configure_acpid.sh | 15 +++++++++++++++ .../definitions/systemvmtemplate/definition.rb | 1 + .../definitions/systemvmtemplate/postinstall.sh | 15 --------------- 3 files changed, 16 insertions(+), 15 deletions(-) create mode 100644 tools/appliance/definitions/systemvmtemplate/configure_acpid.sh diff --git a/tools/appliance/definitions/systemvmtemplate/configure_acpid.sh b/tools/appliance/definitions/systemvmtemplate/configure_acpid.sh new file mode 100644 index 00000000000..70abe30bfc9 --- /dev/null +++ b/tools/appliance/definitions/systemvmtemplate/configure_acpid.sh @@ -0,0 +1,15 @@ +fix_acpid() { + # Fix acpid + mkdir -p /etc/acpi/events + cat >> /etc/acpi/events/power << EOF +event=button/power.* +action=/usr/local/sbin/power.sh "%e" +EOF + cat >> /usr/local/sbin/power.sh << EOF +#!/bin/bash +/sbin/poweroff +EOF + chmod a+x /usr/local/sbin/power.sh +} + +fix_acpid diff --git a/tools/appliance/definitions/systemvmtemplate/definition.rb b/tools/appliance/definitions/systemvmtemplate/definition.rb index be0b4036ce0..a2eb82bec6b 100644 --- a/tools/appliance/definitions/systemvmtemplate/definition.rb +++ b/tools/appliance/definitions/systemvmtemplate/definition.rb @@ -63,6 +63,7 @@ config = { 'configure_locale.sh', 'configure_login.sh', 'postinstall.sh', + 'configure_acpid.sh', 'cleanup.sh', 'configure_networking.sh', 'zerodisk.sh' diff --git a/tools/appliance/definitions/systemvmtemplate/postinstall.sh b/tools/appliance/definitions/systemvmtemplate/postinstall.sh index 893b5214fb1..f2ce1ae6d74 100644 --- a/tools/appliance/definitions/systemvmtemplate/postinstall.sh +++ b/tools/appliance/definitions/systemvmtemplate/postinstall.sh @@ -116,20 +116,6 @@ nameserver 8.8.4.4 EOF } -fix_acpid() { - # Fix acpid - mkdir -p /etc/acpi/events - cat >> /etc/acpi/events/power << EOF -event=button/power.* -action=/usr/local/sbin/power.sh "%e" -EOF - cat >> /usr/local/sbin/power.sh << EOF -#!/bin/bash -/sbin/poweroff -EOF - chmod a+x /usr/local/sbin/power.sh -} - fix_hostname() { # Fix hostname in openssh-server generated keys sed -i "s/root@\(.*\)$/root@$HOSTNAME/g" /etc/ssh/ssh_host_*.pub @@ -164,7 +150,6 @@ EOF do_fixes() { fix_nameserver - fix_acpid fix_hostname fix_conntrackd fix_vhdutil From 3c9f2c72271bdb49b01dd8d9df922440427f05e4 Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 11:19:48 +0200 Subject: [PATCH 17/60] CLOUDSTACK-7143: cleanup configure_acpid.sh code --- .../definitions/systemvmtemplate/configure_acpid.sh | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tools/appliance/definitions/systemvmtemplate/configure_acpid.sh b/tools/appliance/definitions/systemvmtemplate/configure_acpid.sh index 70abe30bfc9..4e986c0d7f4 100644 --- a/tools/appliance/definitions/systemvmtemplate/configure_acpid.sh +++ b/tools/appliance/definitions/systemvmtemplate/configure_acpid.sh @@ -1,5 +1,11 @@ -fix_acpid() { - # Fix acpid +#!/bin/bash + +set -e +set -x + +function configure_acpid() { + grep /usr/local/sbin/power.sh /etc/acpi/events/power && return + mkdir -p /etc/acpi/events cat >> /etc/acpi/events/power << EOF event=button/power.* @@ -12,4 +18,4 @@ EOF chmod a+x /usr/local/sbin/power.sh } -fix_acpid +return 2>/dev/null || configure_acpid From ed5316754761f3dc70c1487fc9ce4074c62c7090 Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 11:21:03 +0200 Subject: [PATCH 18/60] CLOUDSTACK-7143: cleanup cleanup.sh code --- .../definitions/systemvmtemplate/cleanup.sh | 41 +++++++++++++------ 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/tools/appliance/definitions/systemvmtemplate/cleanup.sh b/tools/appliance/definitions/systemvmtemplate/cleanup.sh index 3b1d4bf9f88..af3a1695c6b 100644 --- a/tools/appliance/definitions/systemvmtemplate/cleanup.sh +++ b/tools/appliance/definitions/systemvmtemplate/cleanup.sh @@ -1,16 +1,33 @@ -# Clean up -#apt-get -y remove linux-headers-$(uname -r) build-essential -apt-get -y remove dictionaries-common busybox -apt-get -y autoremove -apt-get autoclean -apt-get clean +#!/bin/bash + +set -e +set -x + +function cleanup_apt() { + #apt-get -y remove linux-headers-$(uname -r) build-essential + apt-get -y remove dictionaries-common busybox + apt-get -y autoremove + apt-get autoclean + apt-get clean +} # Removing leftover leases and persistent rules -echo "cleaning up dhcp leases" -rm /var/lib/dhcp/* +function cleanup_dhcp() { + rm -f /var/lib/dhcp/* +} # Make sure Udev doesn't block our network -echo "cleaning up udev rules" -rm /etc/udev/rules.d/70-persistent-net.rules -rm -rf /dev/.udev/ -rm /lib/udev/rules.d/75-persistent-net-generator.rules +function cleanup_dev() { + echo "cleaning up udev rules" + rm -f /etc/udev/rules.d/70-persistent-net.rules + rm -rf /dev/.udev/ + rm -f /lib/udev/rules.d/75-persistent-net-generator.rules +} + +function cleanup() { + cleanup_apt + cleanup_dhcp + cleanup_dev +} + +return 2>/dev/null || cleanup From 3574dcf54357765e6c1e3389b9ae60f95555dcd4 Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 11:27:16 +0200 Subject: [PATCH 19/60] CLOUDSTACK-7143: split connection tracking config into its own file --- .../systemvmtemplate/configure_conntrack.sh | 19 +++++++++++++++++++ .../systemvmtemplate/definition.rb | 1 + .../systemvmtemplate/postinstall.sh | 19 ------------------- 3 files changed, 20 insertions(+), 19 deletions(-) create mode 100644 tools/appliance/definitions/systemvmtemplate/configure_conntrack.sh diff --git a/tools/appliance/definitions/systemvmtemplate/configure_conntrack.sh b/tools/appliance/definitions/systemvmtemplate/configure_conntrack.sh new file mode 100644 index 00000000000..7fbd33a65be --- /dev/null +++ b/tools/appliance/definitions/systemvmtemplate/configure_conntrack.sh @@ -0,0 +1,19 @@ +# This is actually a bug in the conntrackd package. The comment in the conf file says stats logging is off by default but the parameter is set to on. +# After a couple weeks logrotate will rotate the conntrackd-stats.log file ans start conntracking even if we don't want it to (on non-redundant routers for instance). +fix_conntrackd() { + sed -i '/Stats {/,/}/ s/LogFile on/LogFile off/' /etc/conntrackd/conntrackd.conf + rm -f /var/log/conntrackd-stats.log +} + +# Preload these module otherwise the sysctl settings will not be set, and pasive ftp will not work. +fix_modules() { + cat >> /etc/modules << EOF +nf_conntrack_ipv4 +nf_conntrack +nf_conntrack_ftp +nf_nat_ftp +EOF +} + +fix_conntrackd +fix_modules diff --git a/tools/appliance/definitions/systemvmtemplate/definition.rb b/tools/appliance/definitions/systemvmtemplate/definition.rb index a2eb82bec6b..48c656cdf7b 100644 --- a/tools/appliance/definitions/systemvmtemplate/definition.rb +++ b/tools/appliance/definitions/systemvmtemplate/definition.rb @@ -64,6 +64,7 @@ config = { 'configure_login.sh', 'postinstall.sh', 'configure_acpid.sh', + 'configure_conntrack.sh', 'cleanup.sh', 'configure_networking.sh', 'zerodisk.sh' diff --git a/tools/appliance/definitions/systemvmtemplate/postinstall.sh b/tools/appliance/definitions/systemvmtemplate/postinstall.sh index f2ce1ae6d74..55d658bf79c 100644 --- a/tools/appliance/definitions/systemvmtemplate/postinstall.sh +++ b/tools/appliance/definitions/systemvmtemplate/postinstall.sh @@ -126,34 +126,15 @@ fix_hostname() { sed -i '/127.0.1.1/d' /etc/hosts } -# This is actually a bug in the conntrackd package. The comment in the conf file says stats logging is off by default but the parameter is set to on. -# After a couple weeks logrotate will rotate the conntrackd-stats.log file ans start conntracking even if we don't want it to (on non-redundant routers for instance). -fix_conntrackd() { - sed -i '/Stats {/,/}/ s/LogFile on/LogFile off/' /etc/conntrackd/conntrackd.conf - rm -f /var/log/conntrackd-stats.log -} - fix_vhdutil() { wget --no-check-certificate http://download.cloud.com.s3.amazonaws.com/tools/vhd-util -O /bin/vhd-util chmod a+x /bin/vhd-util } -# Preload these module otherwise the sysctl settings will not be set, and pasive ftp will not work. -fix_modules() { - cat >> /etc/modules << EOF -nf_conntrack_ipv4 -nf_conntrack -nf_conntrack_ftp -nf_nat_ftp -EOF -} - do_fixes() { fix_nameserver fix_hostname - fix_conntrackd fix_vhdutil - fix_modules } configure_apache2() { From a533de7593a9605f42a2a84e5a15e36c4b911d75 Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 11:41:37 +0200 Subject: [PATCH 20/60] CLOUDSTACK-7143: move networking logic from postinstall.sh to configure_networking.sh --- .../systemvmtemplate/configure_networking.sh | 23 +++++++++++++++++++ .../systemvmtemplate/definition.rb | 2 +- .../systemvmtemplate/postinstall.sh | 20 ---------------- 3 files changed, 24 insertions(+), 21 deletions(-) diff --git a/tools/appliance/definitions/systemvmtemplate/configure_networking.sh b/tools/appliance/definitions/systemvmtemplate/configure_networking.sh index eca0d5c1c45..4d1fcced77c 100644 --- a/tools/appliance/definitions/systemvmtemplate/configure_networking.sh +++ b/tools/appliance/definitions/systemvmtemplate/configure_networking.sh @@ -1,2 +1,25 @@ +HOSTNAME=systemvm + echo "Adding a 2 sec delay to the interface up, to make the dhclient happy" echo "pre-up sleep 2" >> /etc/network/interfaces + +fix_nameserver() { + # Replace /etc/resolv.conf also + cat > /etc/resolv.conf << EOF +nameserver 8.8.8.8 +nameserver 8.8.4.4 +EOF +} + +fix_hostname() { + # Fix hostname in openssh-server generated keys + sed -i "s/root@\(.*\)$/root@$HOSTNAME/g" /etc/ssh/ssh_host_*.pub + # Fix hostname to override one provided by dhcp during vm build + echo "$HOSTNAME" > /etc/hostname + hostname $HOSTNAME + # Delete entry in /etc/hosts derived from dhcp + sed -i '/127.0.1.1/d' /etc/hosts +} + +fix_hostname +fix_nameserver diff --git a/tools/appliance/definitions/systemvmtemplate/definition.rb b/tools/appliance/definitions/systemvmtemplate/definition.rb index 48c656cdf7b..6daa7ff709b 100644 --- a/tools/appliance/definitions/systemvmtemplate/definition.rb +++ b/tools/appliance/definitions/systemvmtemplate/definition.rb @@ -62,11 +62,11 @@ config = { 'configure_grub.sh', 'configure_locale.sh', 'configure_login.sh', + 'configure_networking.sh', 'postinstall.sh', 'configure_acpid.sh', 'configure_conntrack.sh', 'cleanup.sh', - 'configure_networking.sh', 'zerodisk.sh' ], :postinstall_timeout => '10000' diff --git a/tools/appliance/definitions/systemvmtemplate/postinstall.sh b/tools/appliance/definitions/systemvmtemplate/postinstall.sh index 55d658bf79c..7ce032c111e 100644 --- a/tools/appliance/definitions/systemvmtemplate/postinstall.sh +++ b/tools/appliance/definitions/systemvmtemplate/postinstall.sh @@ -108,32 +108,12 @@ install_packages() { apt-get --no-install-recommends -q -y --force-yes install radvd } -fix_nameserver() { - # Replace /etc/resolv.conf also - cat > /etc/resolv.conf << EOF -nameserver 8.8.8.8 -nameserver 8.8.4.4 -EOF -} - -fix_hostname() { - # Fix hostname in openssh-server generated keys - sed -i "s/root@\(.*\)$/root@$HOSTNAME/g" /etc/ssh/ssh_host_*.pub - # Fix hostname to override one provided by dhcp during vm build - echo "$HOSTNAME" > /etc/hostname - hostname $HOSTNAME - # Delete entry in /etc/hosts derived from dhcp - sed -i '/127.0.1.1/d' /etc/hosts -} - fix_vhdutil() { wget --no-check-certificate http://download.cloud.com.s3.amazonaws.com/tools/vhd-util -O /bin/vhd-util chmod a+x /bin/vhd-util } do_fixes() { - fix_nameserver - fix_hostname fix_vhdutil } From 71c913c658583844a57c53ad5b997b4f1b414ea2 Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 11:42:27 +0200 Subject: [PATCH 21/60] CLOUDSTACK-7143: cleanup configure_networking.sh code --- .../systemvmtemplate/configure_networking.sh | 47 ++++++++++++++----- 1 file changed, 34 insertions(+), 13 deletions(-) diff --git a/tools/appliance/definitions/systemvmtemplate/configure_networking.sh b/tools/appliance/definitions/systemvmtemplate/configure_networking.sh index 4d1fcced77c..1d9f6b9b962 100644 --- a/tools/appliance/definitions/systemvmtemplate/configure_networking.sh +++ b/tools/appliance/definitions/systemvmtemplate/configure_networking.sh @@ -1,25 +1,46 @@ +#!/bin/bash + +set -e +set -x + HOSTNAME=systemvm -echo "Adding a 2 sec delay to the interface up, to make the dhclient happy" -echo "pre-up sleep 2" >> /etc/network/interfaces +# Adding a 2 sec delay to the interface up, to make the dhclient happy +function set_interface_sleep() { + grep "pre-up sleep 2" /etc/network/interfaces && return + + echo "pre-up sleep 2" >> /etc/network/interfaces +} + +function configure_resolv_conf() { + grep 8.8.8.8 /etc/resolv.conf && grep 8.8.4.4 /etc/resolv.conf && return -fix_nameserver() { - # Replace /etc/resolv.conf also cat > /etc/resolv.conf << EOF nameserver 8.8.8.8 nameserver 8.8.4.4 EOF } -fix_hostname() { - # Fix hostname in openssh-server generated keys - sed -i "s/root@\(.*\)$/root@$HOSTNAME/g" /etc/ssh/ssh_host_*.pub - # Fix hostname to override one provided by dhcp during vm build - echo "$HOSTNAME" > /etc/hostname - hostname $HOSTNAME - # Delete entry in /etc/hosts derived from dhcp +# Delete entry in /etc/hosts derived from dhcp +function delete_dhcp_ip() { + result=$(grep 127.0.1.1 /etc/hosts || true) + [ "${result}" == "" ] && return + sed -i '/127.0.1.1/d' /etc/hosts } -fix_hostname -fix_nameserver +function configure_hostname() { + sed -i "s/root@\(.*\)$/root@$HOSTNAME/g" /etc/ssh/ssh_host_*.pub + + echo "$HOSTNAME" > /etc/hostname + hostname $HOSTNAME +} + +function configure_networking() { + set_interface_sleep + configure_resolv_conf + delete_dhcp_ip + configure_hostname +} + +return 2>/dev/null || configure_networking From 06c1c16dc38a615dfead07dd78179b0f0dfb56f2 Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 11:46:02 +0200 Subject: [PATCH 22/60] CLOUDSTACK-7143: move install_packages from postinstall.sh to its own file --- .../systemvmtemplate/definition.rb | 3 +- .../install_systemvm_packages.sh | 88 ++++++++++++++++++ .../systemvmtemplate/postinstall.sh | 90 ------------------- 3 files changed, 90 insertions(+), 91 deletions(-) create mode 100644 tools/appliance/definitions/systemvmtemplate/install_systemvm_packages.sh diff --git a/tools/appliance/definitions/systemvmtemplate/definition.rb b/tools/appliance/definitions/systemvmtemplate/definition.rb index 6daa7ff709b..556b8f2737f 100644 --- a/tools/appliance/definitions/systemvmtemplate/definition.rb +++ b/tools/appliance/definitions/systemvmtemplate/definition.rb @@ -63,8 +63,9 @@ config = { 'configure_locale.sh', 'configure_login.sh', 'configure_networking.sh', - 'postinstall.sh', 'configure_acpid.sh', + 'install_systemvm_packages.sh', + 'postinstall.sh', 'configure_conntrack.sh', 'cleanup.sh', 'zerodisk.sh' diff --git a/tools/appliance/definitions/systemvmtemplate/install_systemvm_packages.sh b/tools/appliance/definitions/systemvmtemplate/install_systemvm_packages.sh new file mode 100644 index 00000000000..daa0efeabba --- /dev/null +++ b/tools/appliance/definitions/systemvmtemplate/install_systemvm_packages.sh @@ -0,0 +1,88 @@ +install_packages() { + DEBIAN_FRONTEND=noninteractive + DEBIAN_PRIORITY=critical + local arch=`dpkg --print-architecture` + + # Basic packages + apt-get --no-install-recommends -q -y --force-yes install rsyslog logrotate cron chkconfig insserv net-tools ifupdown vim-tiny netbase iptables + apt-get --no-install-recommends -q -y --force-yes install openssh-server openssl e2fsprogs dhcp3-client tcpdump socat wget + # apt-get --no-install-recommends -q -y --force-yes install grub-legacy + apt-get --no-install-recommends -q -y --force-yes install python bzip2 sed gawk diffutils grep gzip less tar telnet ftp rsync traceroute psmisc lsof procps inetutils-ping iputils-arping httping + apt-get --no-install-recommends -q -y --force-yes install dnsutils zip unzip ethtool uuid file iproute acpid virt-what sudo + + # sysstat + echo 'sysstat sysstat/enable boolean true' | debconf-set-selections + apt-get --no-install-recommends -q -y --force-yes install sysstat + # apache + apt-get --no-install-recommends -q -y --force-yes install apache2 ssl-cert + + # dnsmasq + apt-get --no-install-recommends -q -y --force-yes install dnsmasq dnsmasq-utils + # nfs client + apt-get --no-install-recommends -q -y --force-yes install nfs-common + # nfs irqbalance + apt-get --no-install-recommends -q -y --force-yes install irqbalance + + # cifs client + apt-get --no-install-recommends -q -y --force-yes install samba-common + apt-get --no-install-recommends -q -y --force-yes install cifs-utils + + # vpn stuff + apt-get --no-install-recommends -q -y --force-yes install xl2tpd bcrelay ppp ipsec-tools tdb-tools + echo "openswan openswan/install_x509_certificate boolean false" | debconf-set-selections + echo "openswan openswan/install_x509_certificate seen true" | debconf-set-selections + apt-get --no-install-recommends -q -y --force-yes install openswan=1:2.6.37-3 + + # xenstore utils + apt-get --no-install-recommends -q -y --force-yes install xenstore-utils libxenstore3.0 + # keepalived and conntrackd for redundant router + apt-get --no-install-recommends -q -y --force-yes install keepalived conntrackd ipvsadm libnetfilter-conntrack3 libnl1 + # ipcalc + apt-get --no-install-recommends -q -y --force-yes install ipcalc + apt-get update + # java + apt-get --no-install-recommends -q -y --force-yes install openjdk-7-jre-headless + + echo "iptables-persistent iptables-persistent/autosave_v4 boolean true" | debconf-set-selections + echo "iptables-persistent iptables-persistent/autosave_v6 boolean true" | debconf-set-selections + apt-get --no-install-recommends -q -y --force-yes install iptables-persistent + + # Hyperv kvp daemon - 64bit only + if [ "${arch}" == "amd64" ]; then + # Download the hv kvp daemon + wget http://people.apache.org/~rajeshbattala/hv-kvp-daemon_3.1_amd64.deb + dpkg -i hv-kvp-daemon_3.1_amd64.deb + fi + + #libraries required for rdp client (Hyper-V) + apt-get --no-install-recommends -q -y --force-yes install libtcnative-1 libssl-dev libapr1-dev + + # vmware tools + apt-get --no-install-recommends -q -y --force-yes install open-vm-tools + # commented installaion of vmware-tools as we are using the opensource open-vm-tools: + # apt-get --no-install-recommends -q -y --force-yes install build-essential linux-headers-`uname -r` + # df -h + # PREVDIR=$PWD + # cd /opt + # wget http://people.apache.org/~bhaisaab/cloudstack/VMwareTools-9.2.1-818201.tar.gz + # tar xzf VMwareTools-9.2.1-818201.tar.gz + # rm VMwareTools-*.tar.gz + # cd vmware-tools-distrib + # ./vmware-install.pl -d + # cd $PREV + # rm -fr /opt/vmware-tools-distrib + # apt-get -q -y --force-yes purge build-essential + + apt-get --no-install-recommends -q -y --force-yes install haproxy + + #32 bit architecture support:: not required for 32 bit template + if [ "${arch}" != "i386" ]; then + dpkg --add-architecture i386 + apt-get update + apt-get --no-install-recommends -q -y --force-yes install links:i386 libuuid1:i386 + fi + + apt-get --no-install-recommends -q -y --force-yes install radvd +} + +install_packages diff --git a/tools/appliance/definitions/systemvmtemplate/postinstall.sh b/tools/appliance/definitions/systemvmtemplate/postinstall.sh index 7ce032c111e..cf995dc58f5 100644 --- a/tools/appliance/definitions/systemvmtemplate/postinstall.sh +++ b/tools/appliance/definitions/systemvmtemplate/postinstall.sh @@ -21,93 +21,6 @@ ROOTPW=password HOSTNAME=systemvm CLOUDSTACK_RELEASE=4.5.0 -install_packages() { - DEBIAN_FRONTEND=noninteractive - DEBIAN_PRIORITY=critical - local arch=`dpkg --print-architecture` - - # Basic packages - apt-get --no-install-recommends -q -y --force-yes install rsyslog logrotate cron chkconfig insserv net-tools ifupdown vim-tiny netbase iptables - apt-get --no-install-recommends -q -y --force-yes install openssh-server openssl e2fsprogs dhcp3-client tcpdump socat wget - # apt-get --no-install-recommends -q -y --force-yes install grub-legacy - apt-get --no-install-recommends -q -y --force-yes install python bzip2 sed gawk diffutils grep gzip less tar telnet ftp rsync traceroute psmisc lsof procps inetutils-ping iputils-arping httping - apt-get --no-install-recommends -q -y --force-yes install dnsutils zip unzip ethtool uuid file iproute acpid virt-what sudo - - # sysstat - echo 'sysstat sysstat/enable boolean true' | debconf-set-selections - apt-get --no-install-recommends -q -y --force-yes install sysstat - # apache - apt-get --no-install-recommends -q -y --force-yes install apache2 ssl-cert - - # dnsmasq - apt-get --no-install-recommends -q -y --force-yes install dnsmasq dnsmasq-utils - # nfs client - apt-get --no-install-recommends -q -y --force-yes install nfs-common - # nfs irqbalance - apt-get --no-install-recommends -q -y --force-yes install irqbalance - - # cifs client - apt-get --no-install-recommends -q -y --force-yes install samba-common - apt-get --no-install-recommends -q -y --force-yes install cifs-utils - - # vpn stuff - apt-get --no-install-recommends -q -y --force-yes install xl2tpd bcrelay ppp ipsec-tools tdb-tools - echo "openswan openswan/install_x509_certificate boolean false" | debconf-set-selections - echo "openswan openswan/install_x509_certificate seen true" | debconf-set-selections - apt-get --no-install-recommends -q -y --force-yes install openswan=1:2.6.37-3 - - # xenstore utils - apt-get --no-install-recommends -q -y --force-yes install xenstore-utils libxenstore3.0 - # keepalived and conntrackd for redundant router - apt-get --no-install-recommends -q -y --force-yes install keepalived conntrackd ipvsadm libnetfilter-conntrack3 libnl1 - # ipcalc - apt-get --no-install-recommends -q -y --force-yes install ipcalc - apt-get update - # java - apt-get --no-install-recommends -q -y --force-yes install openjdk-7-jre-headless - - echo "iptables-persistent iptables-persistent/autosave_v4 boolean true" | debconf-set-selections - echo "iptables-persistent iptables-persistent/autosave_v6 boolean true" | debconf-set-selections - apt-get --no-install-recommends -q -y --force-yes install iptables-persistent - - # Hyperv kvp daemon - 64bit only - if [ "${arch}" == "amd64" ]; then - # Download the hv kvp daemon - wget http://people.apache.org/~rajeshbattala/hv-kvp-daemon_3.1_amd64.deb - dpkg -i hv-kvp-daemon_3.1_amd64.deb - fi - - #libraries required for rdp client (Hyper-V) - apt-get --no-install-recommends -q -y --force-yes install libtcnative-1 libssl-dev libapr1-dev - - # vmware tools - apt-get --no-install-recommends -q -y --force-yes install open-vm-tools - # commented installaion of vmware-tools as we are using the opensource open-vm-tools: - # apt-get --no-install-recommends -q -y --force-yes install build-essential linux-headers-`uname -r` - # df -h - # PREVDIR=$PWD - # cd /opt - # wget http://people.apache.org/~bhaisaab/cloudstack/VMwareTools-9.2.1-818201.tar.gz - # tar xzf VMwareTools-9.2.1-818201.tar.gz - # rm VMwareTools-*.tar.gz - # cd vmware-tools-distrib - # ./vmware-install.pl -d - # cd $PREV - # rm -fr /opt/vmware-tools-distrib - # apt-get -q -y --force-yes purge build-essential - - apt-get --no-install-recommends -q -y --force-yes install haproxy - - #32 bit architecture support:: not required for 32 bit template - if [ "${arch}" != "i386" ]; then - dpkg --add-architecture i386 - apt-get update - apt-get --no-install-recommends -q -y --force-yes install links:i386 libuuid1:i386 - fi - - apt-get --no-install-recommends -q -y --force-yes install radvd -} - fix_vhdutil() { wget --no-check-certificate http://download.cloud.com.s3.amazonaws.com/tools/vhd-util -O /bin/vhd-util chmod a+x /bin/vhd-util @@ -176,9 +89,6 @@ do_signature() { begin=$(date +%s) -echo "*************INSTALLING PACKAGES********************" -install_packages -echo "*************DONE INSTALLING PACKAGES********************" configure_services configure_apache2 echo "*************DONE SETTING UP SERVICES********************" From 3d4e4a6e09a37711169e7558af0ec53e11e969e7 Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 12:02:13 +0200 Subject: [PATCH 23/60] CLOUDSTACK-7143: consider vhdutil a software package --- .../systemvmtemplate/install_systemvm_packages.sh | 6 ++++++ .../definitions/systemvmtemplate/postinstall.sh | 11 ----------- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/tools/appliance/definitions/systemvmtemplate/install_systemvm_packages.sh b/tools/appliance/definitions/systemvmtemplate/install_systemvm_packages.sh index daa0efeabba..06e91b00787 100644 --- a/tools/appliance/definitions/systemvmtemplate/install_systemvm_packages.sh +++ b/tools/appliance/definitions/systemvmtemplate/install_systemvm_packages.sh @@ -1,3 +1,8 @@ +fix_vhdutil() { + wget --no-check-certificate http://download.cloud.com.s3.amazonaws.com/tools/vhd-util -O /bin/vhd-util + chmod a+x /bin/vhd-util +} + install_packages() { DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical @@ -86,3 +91,4 @@ install_packages() { } install_packages +fix_vhdutil diff --git a/tools/appliance/definitions/systemvmtemplate/postinstall.sh b/tools/appliance/definitions/systemvmtemplate/postinstall.sh index cf995dc58f5..e6d5073229d 100644 --- a/tools/appliance/definitions/systemvmtemplate/postinstall.sh +++ b/tools/appliance/definitions/systemvmtemplate/postinstall.sh @@ -21,15 +21,6 @@ ROOTPW=password HOSTNAME=systemvm CLOUDSTACK_RELEASE=4.5.0 -fix_vhdutil() { - wget --no-check-certificate http://download.cloud.com.s3.amazonaws.com/tools/vhd-util -O /bin/vhd-util - chmod a+x /bin/vhd-util -} - -do_fixes() { - fix_vhdutil -} - configure_apache2() { # Enable ssl, rewrite and auth a2enmod ssl rewrite auth_basic auth_digest @@ -92,8 +83,6 @@ begin=$(date +%s) configure_services configure_apache2 echo "*************DONE SETTING UP SERVICES********************" -do_fixes -echo "*************DONE FIXING CONFIGURATION********************" do_signature fin=$(date +%s) From 22617e9496668ddc99941f24f16e100b80cdcdca Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 12:07:53 +0200 Subject: [PATCH 24/60] CLOUDSTACK-7143: reorg of install_systemvm_packages.sh (no logic changes, yet) --- .../install_systemvm_packages.sh | 96 +++++++++++-------- 1 file changed, 55 insertions(+), 41 deletions(-) diff --git a/tools/appliance/definitions/systemvmtemplate/install_systemvm_packages.sh b/tools/appliance/definitions/systemvmtemplate/install_systemvm_packages.sh index 06e91b00787..31087bc0078 100644 --- a/tools/appliance/definitions/systemvmtemplate/install_systemvm_packages.sh +++ b/tools/appliance/definitions/systemvmtemplate/install_systemvm_packages.sh @@ -1,71 +1,79 @@ -fix_vhdutil() { +#!/bin/bash + +set -e +set -x + +function install_vhd_util() { + [[ -f /bin/vhd-util ]] && return + wget --no-check-certificate http://download.cloud.com.s3.amazonaws.com/tools/vhd-util -O /bin/vhd-util chmod a+x /bin/vhd-util } -install_packages() { +function debconf_packages() { + echo 'sysstat sysstat/enable boolean true' | debconf-set-selections + echo "openswan openswan/install_x509_certificate boolean false" | debconf-set-selections + echo "openswan openswan/install_x509_certificate seen true" | debconf-set-selections + echo "iptables-persistent iptables-persistent/autosave_v4 boolean true" | debconf-set-selections + echo "iptables-persistent iptables-persistent/autosave_v6 boolean true" | debconf-set-selections +} + +function install_packages() { DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical local arch=`dpkg --print-architecture` + debconf_packages + install_vhd_util + + local apt_install="apt-get --no-install-recommends -q -y --force-yes install" + # Basic packages - apt-get --no-install-recommends -q -y --force-yes install rsyslog logrotate cron chkconfig insserv net-tools ifupdown vim-tiny netbase iptables - apt-get --no-install-recommends -q -y --force-yes install openssh-server openssl e2fsprogs dhcp3-client tcpdump socat wget - # apt-get --no-install-recommends -q -y --force-yes install grub-legacy - apt-get --no-install-recommends -q -y --force-yes install python bzip2 sed gawk diffutils grep gzip less tar telnet ftp rsync traceroute psmisc lsof procps inetutils-ping iputils-arping httping - apt-get --no-install-recommends -q -y --force-yes install dnsutils zip unzip ethtool uuid file iproute acpid virt-what sudo + ${apt_install} rsyslog logrotate cron chkconfig insserv net-tools ifupdown vim-tiny netbase iptables + ${apt_install} openssh-server openssl e2fsprogs dhcp3-client tcpdump socat wget + # ${apt_install} grub-legacy + ${apt_install} python bzip2 sed gawk diffutils grep gzip less tar telnet ftp rsync traceroute psmisc lsof procps inetutils-ping iputils-arping httping + ${apt_install} dnsutils zip unzip ethtool uuid file iproute acpid virt-what sudo # sysstat - echo 'sysstat sysstat/enable boolean true' | debconf-set-selections - apt-get --no-install-recommends -q -y --force-yes install sysstat + ${apt_install} sysstat # apache - apt-get --no-install-recommends -q -y --force-yes install apache2 ssl-cert + ${apt_install} apache2 ssl-cert # dnsmasq - apt-get --no-install-recommends -q -y --force-yes install dnsmasq dnsmasq-utils + ${apt_install} dnsmasq dnsmasq-utils # nfs client - apt-get --no-install-recommends -q -y --force-yes install nfs-common + ${apt_install} nfs-common # nfs irqbalance - apt-get --no-install-recommends -q -y --force-yes install irqbalance + ${apt_install} irqbalance # cifs client - apt-get --no-install-recommends -q -y --force-yes install samba-common - apt-get --no-install-recommends -q -y --force-yes install cifs-utils + ${apt_install} samba-common + ${apt_install} cifs-utils # vpn stuff - apt-get --no-install-recommends -q -y --force-yes install xl2tpd bcrelay ppp ipsec-tools tdb-tools - echo "openswan openswan/install_x509_certificate boolean false" | debconf-set-selections - echo "openswan openswan/install_x509_certificate seen true" | debconf-set-selections - apt-get --no-install-recommends -q -y --force-yes install openswan=1:2.6.37-3 + ${apt_install} xl2tpd bcrelay ppp ipsec-tools tdb-tools + ${apt_install} openswan=1:2.6.37-3 # xenstore utils - apt-get --no-install-recommends -q -y --force-yes install xenstore-utils libxenstore3.0 + ${apt_install} xenstore-utils libxenstore3.0 # keepalived and conntrackd for redundant router - apt-get --no-install-recommends -q -y --force-yes install keepalived conntrackd ipvsadm libnetfilter-conntrack3 libnl1 + ${apt_install} keepalived conntrackd ipvsadm libnetfilter-conntrack3 libnl1 # ipcalc - apt-get --no-install-recommends -q -y --force-yes install ipcalc + ${apt_install} ipcalc apt-get update # java - apt-get --no-install-recommends -q -y --force-yes install openjdk-7-jre-headless + ${apt_install} openjdk-7-jre-headless - echo "iptables-persistent iptables-persistent/autosave_v4 boolean true" | debconf-set-selections - echo "iptables-persistent iptables-persistent/autosave_v6 boolean true" | debconf-set-selections - apt-get --no-install-recommends -q -y --force-yes install iptables-persistent - - # Hyperv kvp daemon - 64bit only - if [ "${arch}" == "amd64" ]; then - # Download the hv kvp daemon - wget http://people.apache.org/~rajeshbattala/hv-kvp-daemon_3.1_amd64.deb - dpkg -i hv-kvp-daemon_3.1_amd64.deb - fi + ${apt_install} iptables-persistent #libraries required for rdp client (Hyper-V) - apt-get --no-install-recommends -q -y --force-yes install libtcnative-1 libssl-dev libapr1-dev + ${apt_install} libtcnative-1 libssl-dev libapr1-dev # vmware tools - apt-get --no-install-recommends -q -y --force-yes install open-vm-tools + ${apt_install} open-vm-tools # commented installaion of vmware-tools as we are using the opensource open-vm-tools: - # apt-get --no-install-recommends -q -y --force-yes install build-essential linux-headers-`uname -r` + # ${apt_install} build-essential linux-headers-`uname -r` # df -h # PREVDIR=$PWD # cd /opt @@ -78,17 +86,23 @@ install_packages() { # rm -fr /opt/vmware-tools-distrib # apt-get -q -y --force-yes purge build-essential - apt-get --no-install-recommends -q -y --force-yes install haproxy + ${apt_install} haproxy + + # Hyperv kvp daemon - 64bit only + if [ "${arch}" == "amd64" ]; then + # Download the hv kvp daemon + wget http://people.apache.org/~rajeshbattala/hv-kvp-daemon_3.1_amd64.deb + dpkg -i hv-kvp-daemon_3.1_amd64.deb + fi #32 bit architecture support:: not required for 32 bit template if [ "${arch}" != "i386" ]; then dpkg --add-architecture i386 apt-get update - apt-get --no-install-recommends -q -y --force-yes install links:i386 libuuid1:i386 + ${apt_install} links:i386 libuuid1:i386 fi - apt-get --no-install-recommends -q -y --force-yes install radvd + ${apt_install} radvd } -install_packages -fix_vhdutil +return 2>/dev/null || install_packages From c7e6387752c5d4b931ed3f1ea1d39cdefffdbc3e Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 12:17:06 +0200 Subject: [PATCH 25/60] CLOUDSTACK-7143: collapse apt-get install onto one line In theory this _could_ have changed behavior (apt coming up with a different solution, or one of the packages configuring a new apt repository), but in my testing, the end result is the same. --- .../install_systemvm_packages.sh | 82 +++++++------------ 1 file changed, 28 insertions(+), 54 deletions(-) diff --git a/tools/appliance/definitions/systemvmtemplate/install_systemvm_packages.sh b/tools/appliance/definitions/systemvmtemplate/install_systemvm_packages.sh index 31087bc0078..915703b0f52 100644 --- a/tools/appliance/definitions/systemvmtemplate/install_systemvm_packages.sh +++ b/tools/appliance/definitions/systemvmtemplate/install_systemvm_packages.sh @@ -28,51 +28,36 @@ function install_packages() { local apt_install="apt-get --no-install-recommends -q -y --force-yes install" - # Basic packages - ${apt_install} rsyslog logrotate cron chkconfig insserv net-tools ifupdown vim-tiny netbase iptables - ${apt_install} openssh-server openssl e2fsprogs dhcp3-client tcpdump socat wget - # ${apt_install} grub-legacy - ${apt_install} python bzip2 sed gawk diffutils grep gzip less tar telnet ftp rsync traceroute psmisc lsof procps inetutils-ping iputils-arping httping - ${apt_install} dnsutils zip unzip ethtool uuid file iproute acpid virt-what sudo + #32 bit architecture support:: not required for 32 bit template + if [ "${arch}" != "i386" ]; then + dpkg --add-architecture i386 + ${apt_install} links:i386 libuuid1:i386 + fi - # sysstat - ${apt_install} sysstat - # apache - ${apt_install} apache2 ssl-cert + ${apt_install} \ + rsyslog logrotate cron chkconfig insserv net-tools ifupdown vim-tiny netbase iptables \ + openssh-server openssl e2fsprogs dhcp3-client tcpdump socat wget \ + python bzip2 sed gawk diffutils grep gzip less tar telnet ftp rsync traceroute psmisc lsof procps \ + inetutils-ping iputils-arping httping \ + dnsutils zip unzip ethtool uuid file iproute acpid virt-what sudo \ + sysstat \ + apache2 ssl-cert \ + dnsmasq dnsmasq-utils \ + nfs-common irqbalance \ + samba-common cifs-utils \ + xl2tpd bcrelay ppp ipsec-tools tdb-tools \ + openswan=1:2.6.37-3 \ + xenstore-utils libxenstore3.0 \ + keepalived conntrackd ipvsadm libnetfilter-conntrack3 libnl1 \ + ipcalc \ + openjdk-7-jre-headless \ + iptables-persistent \ + libtcnative-1 libssl-dev libapr1-dev \ + open-vm-tools \ + haproxy \ + radvd - # dnsmasq - ${apt_install} dnsmasq dnsmasq-utils - # nfs client - ${apt_install} nfs-common - # nfs irqbalance - ${apt_install} irqbalance - - # cifs client - ${apt_install} samba-common - ${apt_install} cifs-utils - - # vpn stuff - ${apt_install} xl2tpd bcrelay ppp ipsec-tools tdb-tools - ${apt_install} openswan=1:2.6.37-3 - - # xenstore utils - ${apt_install} xenstore-utils libxenstore3.0 - # keepalived and conntrackd for redundant router - ${apt_install} keepalived conntrackd ipvsadm libnetfilter-conntrack3 libnl1 - # ipcalc - ${apt_install} ipcalc - apt-get update - # java - ${apt_install} openjdk-7-jre-headless - - ${apt_install} iptables-persistent - - #libraries required for rdp client (Hyper-V) - ${apt_install} libtcnative-1 libssl-dev libapr1-dev - - # vmware tools - ${apt_install} open-vm-tools - # commented installaion of vmware-tools as we are using the opensource open-vm-tools: + # commented out installation of vmware-tools as we are using the open source open-vm-tools: # ${apt_install} build-essential linux-headers-`uname -r` # df -h # PREVDIR=$PWD @@ -86,23 +71,12 @@ function install_packages() { # rm -fr /opt/vmware-tools-distrib # apt-get -q -y --force-yes purge build-essential - ${apt_install} haproxy - # Hyperv kvp daemon - 64bit only if [ "${arch}" == "amd64" ]; then # Download the hv kvp daemon wget http://people.apache.org/~rajeshbattala/hv-kvp-daemon_3.1_amd64.deb dpkg -i hv-kvp-daemon_3.1_amd64.deb fi - - #32 bit architecture support:: not required for 32 bit template - if [ "${arch}" != "i386" ]; then - dpkg --add-architecture i386 - apt-get update - ${apt_install} links:i386 libuuid1:i386 - fi - - ${apt_install} radvd } return 2>/dev/null || install_packages From 7d9e46425f6a56bd318f572135bf16d409ef96da Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 12:28:32 +0200 Subject: [PATCH 26/60] CLOUDSTACK-7143: rename postinstall.sh to represent what remains --- .../{postinstall.sh => configure_systemvm_services.sh} | 0 tools/appliance/definitions/systemvmtemplate/definition.rb | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename tools/appliance/definitions/systemvmtemplate/{postinstall.sh => configure_systemvm_services.sh} (100%) diff --git a/tools/appliance/definitions/systemvmtemplate/postinstall.sh b/tools/appliance/definitions/systemvmtemplate/configure_systemvm_services.sh similarity index 100% rename from tools/appliance/definitions/systemvmtemplate/postinstall.sh rename to tools/appliance/definitions/systemvmtemplate/configure_systemvm_services.sh diff --git a/tools/appliance/definitions/systemvmtemplate/definition.rb b/tools/appliance/definitions/systemvmtemplate/definition.rb index 556b8f2737f..ac67075956b 100644 --- a/tools/appliance/definitions/systemvmtemplate/definition.rb +++ b/tools/appliance/definitions/systemvmtemplate/definition.rb @@ -65,7 +65,7 @@ config = { 'configure_networking.sh', 'configure_acpid.sh', 'install_systemvm_packages.sh', - 'postinstall.sh', + 'configure_systemvm_services.sh', 'configure_conntrack.sh', 'cleanup.sh', 'zerodisk.sh' From 654c871e755d07bb60cb2d31e42d0384be79ae51 Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 13:03:56 +0200 Subject: [PATCH 27/60] CLOUDSTACK-7143: cleanup configure_systemvm_services.sh code --- .../configure_systemvm_services.sh | 58 +++++++++---------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/tools/appliance/definitions/systemvmtemplate/configure_systemvm_services.sh b/tools/appliance/definitions/systemvmtemplate/configure_systemvm_services.sh index e6d5073229d..c2dcf6a26c2 100644 --- a/tools/appliance/definitions/systemvmtemplate/configure_systemvm_services.sh +++ b/tools/appliance/definitions/systemvmtemplate/configure_systemvm_services.sh @@ -1,3 +1,4 @@ +#!/bin/bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -15,13 +16,12 @@ # specific language governing permissions and limitations # under the License. +set -e set -x -ROOTPW=password -HOSTNAME=systemvm -CLOUDSTACK_RELEASE=4.5.0 +CLOUDSTACK_RELEASE=4.4.0 -configure_apache2() { +function configure_apache2() { # Enable ssl, rewrite and auth a2enmod ssl rewrite auth_basic auth_digest a2ensite default-ssl @@ -30,17 +30,7 @@ configure_apache2() { cp /etc/apache2/sites-available/default-ssl /etc/apache2/sites-available/default-ssl.orig } -configure_services() { - local arch=`dpkg --print-architecture` - mkdir -p /var/www/html - mkdir -p /opt/cloud/bin - mkdir -p /var/cache/cloud - mkdir -p /usr/share/cloud - mkdir -p /usr/local/cloud - mkdir -p /root/.ssh - # Fix haproxy directory issue - mkdir -p /var/lib/haproxy - +function install_cloud_scripts() { # Get config files from master snapshot_url="https://git-wip-us.apache.org/repos/asf?p=cloudstack.git;a=snapshot;h=HEAD;sf=tgz" snapshot_dir="/opt/cloudstack*" @@ -63,12 +53,6 @@ configure_services() { chkconfig cloud-passwd-srvr off chkconfig --add cloud chkconfig cloud off - chkconfig xl2tpd off - # Hyperv kvp daemon - 64bit only - if [ "${arch}" == "amd64" ]; then - chkconfig hv_kvp_daemon off - fi - chkconfig radvd off } do_signature() { @@ -78,14 +62,30 @@ do_signature() { echo "Cloudstack Release $CLOUDSTACK_RELEASE $(date)" > /etc/cloudstack-release } -begin=$(date +%s) +configure_services() { + mkdir -p /var/www/html + mkdir -p /opt/cloud/bin + mkdir -p /var/cache/cloud + mkdir -p /usr/share/cloud + mkdir -p /usr/local/cloud + mkdir -p /root/.ssh -configure_services -configure_apache2 -echo "*************DONE SETTING UP SERVICES********************" -do_signature + # Fix haproxy directory issue + mkdir -p /var/lib/haproxy -fin=$(date +%s) -t=$((fin-begin)) + install_cloud_scripts -echo "Signed systemvm build, finished building systemvm appliance in $t seconds" + chkconfig xl2tpd off + + # Hyperv kvp daemon - 64bit only + local arch=`dpkg --print-architecture` + if [ "${arch}" == "amd64" ]; then + chkconfig hv_kvp_daemon off + fi + chkconfig radvd off + + configure_apache2 + do_signature +} + +return 2>/dev/null || configure_services From f67a379a466430e1158eddeb42de28aa84283947 Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 13:06:34 +0200 Subject: [PATCH 28/60] CLOUDSTACK-7143: tune timeouts to expect about <=~60 min build --- .../definitions/systemvmtemplate/definition.rb | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tools/appliance/definitions/systemvmtemplate/definition.rb b/tools/appliance/definitions/systemvmtemplate/definition.rb index ac67075956b..a4ba2fe967e 100644 --- a/tools/appliance/definitions/systemvmtemplate/definition.rb +++ b/tools/appliance/definitions/systemvmtemplate/definition.rb @@ -26,7 +26,7 @@ config = { :cpu_count => '1', :memory_size => '256', :disk_size => '2500', :disk_format => 'VDI', :hostiocache => 'off', - :iso_download_timeout => '1000', + :iso_download_timeout => '1200', :boot_wait => '10', :boot_cmd_sequence => [ '', @@ -46,9 +46,9 @@ config = { '' ], :kickstart_port => '7122', - :kickstart_timeout => '10000', + :kickstart_timeout => '1200', :kickstart_file => 'preseed.cfg', - :ssh_login_timeout => '10000', + :ssh_login_timeout => '1200', :ssh_user => 'root', :ssh_password => 'password', :ssh_key => '', @@ -57,6 +57,7 @@ config = { :sudo_cmd => "echo '%p'|sudo -S sh '%f'", :shutdown_cmd => 'halt -p', :postinstall_files => [ + # basic minimal vm creation 'build_time.sh', 'apt_upgrade.sh', 'configure_grub.sh', @@ -64,13 +65,15 @@ config = { 'configure_login.sh', 'configure_networking.sh', 'configure_acpid.sh', + # turning it into a systemvm 'install_systemvm_packages.sh', - 'configure_systemvm_services.sh', 'configure_conntrack.sh', + 'configure_systemvm_services.sh', + # cleanup & space-saving 'cleanup.sh', 'zerodisk.sh' ], - :postinstall_timeout => '10000' + :postinstall_timeout => '1200' } config.merge! architectures[arch.to_sym] From 44ceeabb2813bb40a1a5f95a19a514470a71b52f Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 13:08:35 +0200 Subject: [PATCH 29/60] CLOUDSTACK-7143: cleanup zerodisk.sh code --- .../definitions/systemvmtemplate/zerodisk.sh | 36 ++++++++++++------- 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/tools/appliance/definitions/systemvmtemplate/zerodisk.sh b/tools/appliance/definitions/systemvmtemplate/zerodisk.sh index b00f7ae7ccc..1c8fb53a80b 100644 --- a/tools/appliance/definitions/systemvmtemplate/zerodisk.sh +++ b/tools/appliance/definitions/systemvmtemplate/zerodisk.sh @@ -1,15 +1,27 @@ -# Clean up stuff copied in by veewee -rm -fv /root/*.iso -rm -fv /root/base.sh /root/cleanup.sh /root/postinstall.sh /root/zerodisk.sh -rm -fv .veewee_version .veewee_params .vbox_version +#!/bin/bash -echo "Cleaning up" +set -e +set -x + +# clean up stuff copied in by veewee +function cleanup_veewee() { + # this has to be here since it is the last file to run (and we remove ourselves) + rm -fv /root/*.iso + rm -fv /root/{apt_upgrade,build_time,cleanup,install_systemvm_packages,zerodisk}.sh + rm -fv /root/configure_{acpid,conntrack,grub,locale,login,networking,systemvm_services}.sh + rm -fv .veewee_version .veewee_params .vbox_version +} # Zero out the free space to save space in the final image: -for path in / /boot /usr /var /opt /tmp /home -do - dd if=/dev/zero of=$path/zero bs=1M - sync - rm -f $path/zero - echo "Completed zero-ing out disk on $path" -done +function zero_disk() { + cleanup_veewee + + for path in / /boot /usr /var /opt /tmp /home + do + dd if=/dev/zero of=${path}/zero bs=1M || true + sync + rm -f ${path}/zero + done +} + +return 2>/dev/null || zero_disk From d3ecb0099cb8b1121739af52c576c0c5f868e4a9 Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 13:09:14 +0200 Subject: [PATCH 30/60] CLOUDSTACK-7143: cleanup configure_conntrack.sh code --- .../systemvmtemplate/configure_conntrack.sh | 28 ++++++++++++++----- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/tools/appliance/definitions/systemvmtemplate/configure_conntrack.sh b/tools/appliance/definitions/systemvmtemplate/configure_conntrack.sh index 7fbd33a65be..bd431985d30 100644 --- a/tools/appliance/definitions/systemvmtemplate/configure_conntrack.sh +++ b/tools/appliance/definitions/systemvmtemplate/configure_conntrack.sh @@ -1,12 +1,22 @@ -# This is actually a bug in the conntrackd package. The comment in the conf file says stats logging is off by default but the parameter is set to on. -# After a couple weeks logrotate will rotate the conntrackd-stats.log file ans start conntracking even if we don't want it to (on non-redundant routers for instance). -fix_conntrackd() { +#!/bin/bash + +set -e +set -x + +# This is actually a bug in the conntrackd package. The comment in the conf file says stats logging is off by default +# but the parameter is set to on. +# After a couple weeks logrotate will rotate the conntrackd-stats.log file ans start conntracking even if we don't want +# it to (on non-redundant routers for instance). +function disable_conntrack_logging() { + grep "LogFile off" /etc/conntrackd/conntrackd.conf && return + sed -i '/Stats {/,/}/ s/LogFile on/LogFile off/' /etc/conntrackd/conntrackd.conf rm -f /var/log/conntrackd-stats.log } -# Preload these module otherwise the sysctl settings will not be set, and pasive ftp will not work. -fix_modules() { +function load_conntrack_modules() { + grep nf_conntrack_ipv4 /etc/modules && return + cat >> /etc/modules << EOF nf_conntrack_ipv4 nf_conntrack @@ -15,5 +25,9 @@ nf_nat_ftp EOF } -fix_conntrackd -fix_modules +function configure_conntrack() { + disable_conntrack_logging + load_conntrack_modules +} + +return 2>/dev/null || configure_conntrack From de68741f602814c99807afdbd700bcb316760927 Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 13:13:16 +0200 Subject: [PATCH 31/60] CLOUDSTACK-7143: license headers on all the systemvm scripts --- .../definitions/systemvmtemplate/apt_upgrade.sh | 16 ++++++++++++++++ .../definitions/systemvmtemplate/build_time.sh | 16 ++++++++++++++++ .../definitions/systemvmtemplate/cleanup.sh | 16 ++++++++++++++++ .../systemvmtemplate/configure_acpid.sh | 16 ++++++++++++++++ .../systemvmtemplate/configure_conntrack.sh | 16 ++++++++++++++++ .../systemvmtemplate/configure_grub.sh | 16 ++++++++++++++++ .../systemvmtemplate/configure_locale.sh | 16 ++++++++++++++++ .../systemvmtemplate/configure_login.sh | 16 ++++++++++++++++ .../systemvmtemplate/configure_networking.sh | 16 ++++++++++++++++ .../definitions/systemvmtemplate/definition.rb | 17 +++++++++++++++++ .../install_systemvm_packages.sh | 16 ++++++++++++++++ .../definitions/systemvmtemplate/preseed.cfg | 17 +++++++++++++++++ .../definitions/systemvmtemplate/zerodisk.sh | 16 ++++++++++++++++ 13 files changed, 210 insertions(+) diff --git a/tools/appliance/definitions/systemvmtemplate/apt_upgrade.sh b/tools/appliance/definitions/systemvmtemplate/apt_upgrade.sh index 640d572ffe1..af5f51b1a76 100644 --- a/tools/appliance/definitions/systemvmtemplate/apt_upgrade.sh +++ b/tools/appliance/definitions/systemvmtemplate/apt_upgrade.sh @@ -1,4 +1,20 @@ #!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. set -e set -x diff --git a/tools/appliance/definitions/systemvmtemplate/build_time.sh b/tools/appliance/definitions/systemvmtemplate/build_time.sh index d44b9dbb155..c25644bf08f 100644 --- a/tools/appliance/definitions/systemvmtemplate/build_time.sh +++ b/tools/appliance/definitions/systemvmtemplate/build_time.sh @@ -1,4 +1,20 @@ #!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. set -e set -x diff --git a/tools/appliance/definitions/systemvmtemplate/cleanup.sh b/tools/appliance/definitions/systemvmtemplate/cleanup.sh index af3a1695c6b..cd32d5b42aa 100644 --- a/tools/appliance/definitions/systemvmtemplate/cleanup.sh +++ b/tools/appliance/definitions/systemvmtemplate/cleanup.sh @@ -1,4 +1,20 @@ #!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. set -e set -x diff --git a/tools/appliance/definitions/systemvmtemplate/configure_acpid.sh b/tools/appliance/definitions/systemvmtemplate/configure_acpid.sh index 4e986c0d7f4..f2463424f9d 100644 --- a/tools/appliance/definitions/systemvmtemplate/configure_acpid.sh +++ b/tools/appliance/definitions/systemvmtemplate/configure_acpid.sh @@ -1,4 +1,20 @@ #!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. set -e set -x diff --git a/tools/appliance/definitions/systemvmtemplate/configure_conntrack.sh b/tools/appliance/definitions/systemvmtemplate/configure_conntrack.sh index bd431985d30..54ec41b4279 100644 --- a/tools/appliance/definitions/systemvmtemplate/configure_conntrack.sh +++ b/tools/appliance/definitions/systemvmtemplate/configure_conntrack.sh @@ -1,4 +1,20 @@ #!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. set -e set -x diff --git a/tools/appliance/definitions/systemvmtemplate/configure_grub.sh b/tools/appliance/definitions/systemvmtemplate/configure_grub.sh index 7978bec5301..5b6a80aacb9 100644 --- a/tools/appliance/definitions/systemvmtemplate/configure_grub.sh +++ b/tools/appliance/definitions/systemvmtemplate/configure_grub.sh @@ -1,4 +1,20 @@ #!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. set -e set -x diff --git a/tools/appliance/definitions/systemvmtemplate/configure_locale.sh b/tools/appliance/definitions/systemvmtemplate/configure_locale.sh index 8747a571d34..8db7e4e5598 100644 --- a/tools/appliance/definitions/systemvmtemplate/configure_locale.sh +++ b/tools/appliance/definitions/systemvmtemplate/configure_locale.sh @@ -1,4 +1,20 @@ #!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. set -e set -x diff --git a/tools/appliance/definitions/systemvmtemplate/configure_login.sh b/tools/appliance/definitions/systemvmtemplate/configure_login.sh index 680b08a7818..36fccabc5fa 100644 --- a/tools/appliance/definitions/systemvmtemplate/configure_login.sh +++ b/tools/appliance/definitions/systemvmtemplate/configure_login.sh @@ -1,4 +1,20 @@ #!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. set -e set -x diff --git a/tools/appliance/definitions/systemvmtemplate/configure_networking.sh b/tools/appliance/definitions/systemvmtemplate/configure_networking.sh index 1d9f6b9b962..bddcd6c0f3d 100644 --- a/tools/appliance/definitions/systemvmtemplate/configure_networking.sh +++ b/tools/appliance/definitions/systemvmtemplate/configure_networking.sh @@ -1,4 +1,20 @@ #!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. set -e set -x diff --git a/tools/appliance/definitions/systemvmtemplate/definition.rb b/tools/appliance/definitions/systemvmtemplate/definition.rb index a4ba2fe967e..c560e01c120 100644 --- a/tools/appliance/definitions/systemvmtemplate/definition.rb +++ b/tools/appliance/definitions/systemvmtemplate/definition.rb @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + arch = ENV['VM_ARCH'] || 'i386' # diff --git a/tools/appliance/definitions/systemvmtemplate/install_systemvm_packages.sh b/tools/appliance/definitions/systemvmtemplate/install_systemvm_packages.sh index 915703b0f52..d0436b43db3 100644 --- a/tools/appliance/definitions/systemvmtemplate/install_systemvm_packages.sh +++ b/tools/appliance/definitions/systemvmtemplate/install_systemvm_packages.sh @@ -1,4 +1,20 @@ #!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. set -e set -x diff --git a/tools/appliance/definitions/systemvmtemplate/preseed.cfg b/tools/appliance/definitions/systemvmtemplate/preseed.cfg index 293d6534547..cac9fc7dbc9 100644 --- a/tools/appliance/definitions/systemvmtemplate/preseed.cfg +++ b/tools/appliance/definitions/systemvmtemplate/preseed.cfg @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + ### Localization # Locale sets language and country. d-i debian-installer/locale string en_US diff --git a/tools/appliance/definitions/systemvmtemplate/zerodisk.sh b/tools/appliance/definitions/systemvmtemplate/zerodisk.sh index 1c8fb53a80b..64a51eace8c 100644 --- a/tools/appliance/definitions/systemvmtemplate/zerodisk.sh +++ b/tools/appliance/definitions/systemvmtemplate/zerodisk.sh @@ -1,4 +1,20 @@ #!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. set -e set -x From 2832bc47e1030910485b1ae6e4646a8e9f2b28d9 Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 14:50:08 +0200 Subject: [PATCH 32/60] CLOUDSTACK-7143: improve build.sh ergonomics Added a bunch of possible arguments / environment variables, see the new usage function for details (or run ./build.sh help). Creates a new template with a unique name on every invocation, resulting in a new virtualbox instance with a unique name. This makes it possible to build multiple boxes at the same time, in parallel (including for example on a multiuser box), and reduces the chance of failed build results conflicting with each other. Inject the version given on the command line (if any) into the created image. --- tools/appliance/build.sh | 201 +++++++++++++++++++++++++++++++-------- 1 file changed, 159 insertions(+), 42 deletions(-) diff --git a/tools/appliance/build.sh b/tools/appliance/build.sh index 57d2298bd70..5d74b610df7 100755 --- a/tools/appliance/build.sh +++ b/tools/appliance/build.sh @@ -1,4 +1,6 @@ -#!/bin/bash -xl +#!/bin/bash -l +# note: the -l is needed here for bash to always make a login shell and load rvm if it hasn't been loaded +# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -16,10 +18,90 @@ # specific language governing permissions and limitations # under the License. -set -x +# build script which wraps around veewee and virtualbox to create the systemvm template -appliance="${1:-systemvmtemplate}" -arch="i386" +function usage() { + cat </dev/null && usage +echo $@ | grep '\-h' >/dev/null && usage + +# requires 32-bit vhd-util and faketime binaries to be available (even for 64 bit builds) +# Something like (on centos 6.5)... +# * faketime +# wget -q http://bits.xensource.com/oss-xen/release/4.2.0/xen-4.2.0.tar.gz +# sudo yum -y install libuuid.i686 +# cd repo/libfaketime/ +# vim Makefile +# # (tune 32 bit) +# make +# sudo make install +# * vhd-util +# sudo yum -y install python-devel dev86 iasl iasl-devel libuuid libuuid-devel \ +# glib-devel glib2 glib2-devel yajl yajl-devel +# wget -q http://bits.xensource.com/oss-xen/release/4.2.0/xen-4.2.0.tar.gz +# tar xzvf xen-4.2.0.tar.gz +# cd xen-4.2.0/tools/ +# wget https://github.com/citrix-openstack/xenserver-utils/raw/master/blktap2.patch -qO - | patch -p0 +# ./configure --disable-monitors --disable-ocamltools --disable-rombios --disable-seabios +# make +# sudo cp ./blktap2/vhd/lib/libvhd.so.1.0 /usr/lib64/ +# ldconfig +# sudo ldconfig +# sudo cp blktap2/vhd/vhd-util /usr/lib64/cloud/common/scripts/vm/hypervisor/xenserver +# faketime 2010-01-01 vhd-util convert + +set -e + +### +### Configuration +### +# whether to show DEBUG logs +DEBUG="${DEBUG:-}" +# whether to have other commands trace their actions +TRACE="${TRACE:-0}" +JENKINS_HOME=${JENKINS_HOME:-} +if [[ ! -z "${JENKINS_HOME}" ]]; then + DEBUG=1 +fi +VEEWEE_ARGS="${VEEWEE_ARGS:-}" +if [[ "${VEEWEE_ARGS}" == "" && "${TRACE}" == "1" ]]; then + VEEWEE_ARGS="${VEEWEE_ARGS} --debug" +fi +VEEWEE_BUILD_ARGS="${VEEWEE_BUILD_ARGS:-${VEEWEE_ARGS} --nogui --auto}" + +# which veewee definition to use +appliance="${1:-${appliance:-systemvmtemplate}}" + +# optional version tag to put into the image filename +version="${2:-${version:-}}" + +# branch tag to put into the image filename, populated from `git status` if unset +branch="${3:-${branch:-}}" + +# optional (jenkins) build number tag to put into the image filename +BUILD_NUMBER="${4:-${BUILD_NUMBER:-}}" + +arch="${arch:-i386}" if [ "${appliance}" == "systemvm64template" ]; then arch="amd64" export VM_ARCH="${arch}" @@ -27,42 +109,76 @@ if [ "${appliance}" == "systemvm64template" ]; then cp -r definitions/systemvmtemplate definitions/systemvm64template fi -build_date=`date +%Y-%m-%d` - -# set fixed or leave empty to use git to determine -branch= - -if [ -z "$branch" ] ; then - branch=`(git name-rev --no-undefined --name-only HEAD 2>/dev/null || echo unknown) | sed -e 's/remotes\/.*\///g'` +# while building with vbox, we need a quite unique appliance name in order to prevent conflicts with multiple +# concurrent executors on jenkins +if [ -z "${branch}" ] ; then + branch=`(git name-rev --no-undefined --name-only HEAD 2>/dev/null || echo unknown) | sed -e 's/remotes\/.*\///g'` fi -rootdir=$PWD +branch_tag= +if [ ! -z "${branch}" ]; then + branch_tag="-${branch}" +fi + +version_tag= +if [ ! -z "${version}" ]; then + if [ ! -z "${BUILD_NUMBER}" ]; then + version="${version}.${BUILD_NUMBER}" + fi + version_tag="-${version}" +elif [ ! -z "${BUILD_NUMBER}" ]; then + version="${BUILD_NUMBER}" + version_tag="-${BUILD_NUMBER}" +fi + +appliance_build_name=${appliance}${branch_tag}${version_tag} + +# how to tell sed to use extended regular expressions +os=`uname` +sed_regex_option="-E" +if [ "${os}" == "Linux" ]; then + sed_regex_option="-r" +fi + +# logging support +if [[ "${DEBUG}" == "1" ]]; then + set -x +fi + +# Create custom template definition +if [ "${appliance}" != "${appliance_build_name}" ]; then + cp -r "definitions/${appliance}" "definitions/${appliance_build_name}" + set +e + sed ${sed_regex_option} -i -e "s/^CLOUDSTACK_RELEASE=.+/CLOUDSTACK_RELEASE=${version}/" \ + "definitions/${appliance_build_name}/configure_systemvm_services.sh" + set -e +fi # Initialize veewee and dependencies bundle # Clean and start building the appliance -bundle exec veewee vbox destroy $appliance -bundle exec veewee vbox build $appliance --nogui --auto -bundle exec veewee vbox halt $appliance +bundle exec veewee vbox destroy ${appliance_build_name} ${VEEWEE_ARGS} +bundle exec veewee vbox build ${appliance_build_name} ${VEEWEE_BUILD_ARGS} +bundle exec veewee vbox halt ${appliance_build_name} ${VEEWEE_ARGS} -while [[ `vboxmanage list runningvms | grep $appliance | wc -l` -ne 0 ]]; +while [[ `vboxmanage list runningvms | grep ${appliance_build_name} | wc -l` -ne 0 ]]; do - echo "Waiting for $appliance to shutdown" + echo "Waiting for ${appliance_build_name} to shutdown" sleep 2; done # Get appliance uuids -machine_uuid=`vboxmanage showvminfo $appliance | grep UUID | head -1 | awk '{print $2}'` -hdd_uuid=`vboxmanage showvminfo $appliance | grep vdi | head -1 | awk '{print $8}' | cut -d ')' -f 1` -hdd_path=`vboxmanage list hdds | grep "$appliance\/" | grep vdi | cut -c 14- | sed 's/^ *//'` +machine_uuid=`vboxmanage showvminfo ${appliance_build_name} | grep UUID | head -1 | awk '{print $2}'` +hdd_uuid=`vboxmanage showvminfo ${appliance_build_name} | grep vdi | head -1 | awk '{print $8}' | cut -d ')' -f 1` +hdd_path=`vboxmanage list hdds | grep "${appliance_build_name}\/" | grep vdi | cut -c 14- | sed 's/^ *//'` # Remove any shared folder -shared_folders=`vboxmanage showvminfo $appliance | grep Name | grep Host` +shared_folders=`vboxmanage showvminfo ${appliance_build_name} | grep Name | grep Host` while [ "$shared_folders" != "" ] do - vboxmanage sharedfolder remove $appliance --name "`echo $shared_folders | head -1 | cut -c 8- | cut -d \' -f 1`" - shared_folders=`vboxmanage showvminfo $appliance | grep Name | grep Host` + vboxmanage sharedfolder remove ${appliance_build_name} --name "`echo $shared_folders | head -1 | cut -c 8- | cut -d \' -f 1`" + shared_folders=`vboxmanage showvminfo ${appliance_build_name} | grep Name | grep Host` done # Compact the virtual hdd @@ -78,12 +194,12 @@ if [ $? == 0 ]; then set -e vboxmanage internalcommands converttoraw -format vdi "$hdd_path" img.raw vhd-util convert -s 0 -t 1 -i img.raw -o stagefixed.vhd - faketime '2010-01-01' vhd-util convert -s 1 -t 2 -i stagefixed.vhd -o $appliance-$branch-xen.vhd + faketime '2010-01-01' vhd-util convert -s 1 -t 2 -i stagefixed.vhd -o ${appliance_build_name}-xen.vhd rm *.bak - bzip2 $appliance-$branch-xen.vhd - echo "$appliance exported for XenServer: dist/$appliance-$branch-xen.vhd.bz2" + bzip2 ${appliance_build_name}-xen.vhd + echo "${appliance_build_name} exported for XenServer: dist/${appliance_build_name}-xen.vhd.bz2" else - echo "** Skipping $appliance export for XenServer: faketime or vhd-util command is missing. **" + echo "** Skipping ${appliance_build_name} export for XenServer: faketime or vhd-util command is missing. **" echo "** faketime source code is available from https://github.com/wolfcw/libfaketime **" fi @@ -92,27 +208,28 @@ set -e # Export for KVM vboxmanage internalcommands converttoraw -format vdi "$hdd_path" raw.img -qemu-img convert -f raw -c -O qcow2 raw.img $appliance-$branch-kvm.qcow2 +qemu-img convert -f raw -c -O qcow2 raw.img ${appliance_build_name}-kvm.qcow2 rm raw.img -bzip2 $appliance-$branch-kvm.qcow2 -echo "$appliance exported for KVM: dist/$appliance-$branch-kvm.qcow2.bz2" +bzip2 ${appliance_build_name}-kvm.qcow2 +echo "${appliance_build_name} exported for KVM: dist/${appliance_build_name}-kvm.qcow2.bz2" # Export both ova and vmdk for VMWare -vboxmanage clonehd $hdd_uuid $appliance-$branch-vmware.vmdk --format VMDK -bzip2 $appliance-$branch-vmware.vmdk -echo "$appliance exported for VMWare: dist/$appliance-$branch-vmware.vmdk.bz2" -vboxmanage export $machine_uuid --output $appliance-$branch-vmware.ovf -mv $appliance-$branch-vmware.ovf $appliance-$branch-vmware.ovf-orig -java -cp convert Convert convert_ovf_vbox_to_esx.xslt $appliance-$branch-vmware.ovf-orig $appliance-$branch-vmware.ovf -tar -cf $appliance-$branch-vmware.ova $appliance-$branch-vmware.ovf $appliance-$branch-vmware-disk[0-9].vmdk -rm -f $appliance-$branch-vmware.ovf $appliance-$branch-vmware.ovf-orig $appliance-$branch-vmware-disk[0-9].vmdk -echo "$appliance exported for VMWare: dist/$appliance-$branch-vmware.ova" +vboxmanage clonehd $hdd_uuid ${appliance_build_name}-vmware.vmdk --format VMDK +bzip2 ${appliance_build_name}-vmware.vmdk +echo "${appliance_build_name} exported for VMWare: dist/${appliance_build_name}-vmware.vmdk.bz2" +vboxmanage export $machine_uuid --output ${appliance_build_name}-vmware.ovf +mv ${appliance_build_name}-vmware.ovf ${appliance_build_name}-vmware.ovf-orig +java -cp convert Convert convert_ovf_vbox_to_esx.xslt ${appliance_build_name}-vmware.ovf-orig ${appliance_build_name}-vmware.ovf +tar -cf ${appliance_build_name}-vmware.ova ${appliance_build_name}-vmware.ovf ${appliance_build_name}-vmware-disk[0-9].vmdk +rm -f ${appliance_build_name}-vmware.ovf ${appliance_build_name}-vmware.ovf-orig ${appliance_build_name}-vmware-disk[0-9].vmdk +echo "${appliance_build_name} exported for VMWare: dist/${appliance_build_name}-vmware.ova" # Export for HyperV -vboxmanage clonehd $hdd_uuid $appliance-$branch-hyperv.vhd --format VHD +vboxmanage clonehd $hdd_uuid ${appliance_build_name}-hyperv.vhd --format VHD # HyperV doesn't support import a zipped image from S3, but we create a zipped version to save space on the jenkins box -zip $appliance-$branch-hyperv.vhd.zip $appliance-$branch-hyperv.vhd -echo "$appliance exported for HyperV: dist/$appliance-$branch-hyperv.vhd" +zip ${appliance_build_name}-hyperv.vhd.zip ${appliance_build_name}-hyperv.vhd +echo "${appliance_build_name} exported for HyperV: dist/${appliance_build_name}-hyperv.vhd" mv *-hyperv.vhd *-hyperv.vhd.zip *.bz2 *.ova dist/ +rm -rf "definitions/${appliance_build_name}" From 18364216d8f023b90dbddb0fa6348f657a8038dc Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 15:25:40 +0200 Subject: [PATCH 33/60] CLOUDSTACK-7143: improve build.sh failure handling and recovery In particular, this refactoring allows the use of 'set -e' to exit early on error. Previously the script would continue for a while when encountering a problem, stuttering sometimes to (almost) completion, producing partial or no results. Added a bash on EXIT trap which runs add_on_exit cleanup code in the reverse order it was specified. Resource cleanup is now co-located with resource definition. Added color-coded logging. Made most of the hypervisor-specific exports optional. This script now works on Mac OS X. --- tools/appliance/build.sh | 318 +++++++++++++++++++++++++++++---------- 1 file changed, 241 insertions(+), 77 deletions(-) diff --git a/tools/appliance/build.sh b/tools/appliance/build.sh index 5d74b610df7..39bcba4bbfa 100755 --- a/tools/appliance/build.sh +++ b/tools/appliance/build.sh @@ -23,7 +23,7 @@ function usage() { cat <&2 +} + +function error() { + log ERROR $@ + exit 1 +} + +# cleanup code support +declare -a on_exit_items + +function on_exit() { + for (( i=${#on_exit_items[@]}-1 ; i>=0 ; i-- )) ; do + sleep 2 + log DEBUG "on_exit: ${on_exit_items[i]}" + eval ${on_exit_items[i]} + done +} + +function add_on_exit() { + local n=${#on_exit_items[*]} + on_exit_items[${n}]="$*" + if [ ${n} -eq 0 ]; then + log DEBUG "Setting trap" + trap on_exit EXIT + fi +} + +# retry code support +function retry() { + local times=$1 + shift + local count=0 + while [ ${count} -lt ${times} ]; do + "$@" && break + count=$(( $count + 1 )) + sleep ${count} + done + + if [ ${count} -eq ${times} ]; then + error "Failed ${times} times: $@" + fi +} + +### +### Script logic +### + +function create_definition() { + if [ "${appliance}" != "${appliance_build_name}" ]; then + cp -r "definitions/${appliance}" "definitions/${appliance_build_name}" + set +e + sed ${sed_regex_option} -i -e "s/^CLOUDSTACK_RELEASE=.+/CLOUDSTACK_RELEASE=${version}/" \ + "definitions/${appliance_build_name}/postinstall.sh" + set -e + add_on_exit rm -rf "definitions/${appliance_build_name}" + fi +} + +function prepare() { + log INFO "preparing for build" + bundle + rm -rf dist *.ova *.vhd *.vdi *.qcow* *.bz2 *.vmdk *.ovf + mkdir dist +} + +function veewee_destroy() { + log INFO "destroying existing veewee image, if any" set +e - sed ${sed_regex_option} -i -e "s/^CLOUDSTACK_RELEASE=.+/CLOUDSTACK_RELEASE=${version}/" \ - "definitions/${appliance_build_name}/configure_systemvm_services.sh" + bundle exec veewee vbox destroy "${appliance_build_name}" ${VEEWEE_ARGS} set -e -fi +} -# Initialize veewee and dependencies -bundle +function veewee_build() { + log INFO "building new image with veewee" + bundle exec veewee vbox build "${appliance_build_name}" ${VEEWEE_BUILD_ARGS} + bundle exec veewee vbox halt "${appliance_build_name}" ${VEEWEE_ARGS} +} -# Clean and start building the appliance -bundle exec veewee vbox destroy ${appliance_build_name} ${VEEWEE_ARGS} -bundle exec veewee vbox build ${appliance_build_name} ${VEEWEE_BUILD_ARGS} -bundle exec veewee vbox halt ${appliance_build_name} ${VEEWEE_ARGS} +function check_appliance_shutdown() { + log INFO "waiting for veewee appliance to shut down..." + ! (vboxmanage list runningvms | grep "${appliance_build_name}") + local result=$? + if [ ${result} -eq 0 ]; then + log INFO "...veewee appliance shut down ok" + else + log INFO "...veewee appliance still running" + fi + return ${result} +} -while [[ `vboxmanage list runningvms | grep ${appliance_build_name} | wc -l` -ne 0 ]]; -do - echo "Waiting for ${appliance_build_name} to shutdown" - sleep 2; -done - -# Get appliance uuids -machine_uuid=`vboxmanage showvminfo ${appliance_build_name} | grep UUID | head -1 | awk '{print $2}'` -hdd_uuid=`vboxmanage showvminfo ${appliance_build_name} | grep vdi | head -1 | awk '{print $8}' | cut -d ')' -f 1` -hdd_path=`vboxmanage list hdds | grep "${appliance_build_name}\/" | grep vdi | cut -c 14- | sed 's/^ *//'` - -# Remove any shared folder -shared_folders=`vboxmanage showvminfo ${appliance_build_name} | grep Name | grep Host` -while [ "$shared_folders" != "" ] -do - vboxmanage sharedfolder remove ${appliance_build_name} --name "`echo $shared_folders | head -1 | cut -c 8- | cut -d \' -f 1`" - shared_folders=`vboxmanage showvminfo ${appliance_build_name} | grep Name | grep Host` -done - -# Compact the virtual hdd -vboxmanage modifyhd $hdd_uuid --compact - -# Start exporting -rm -fr dist *.ova *.vhd *.vdi *.qcow* *.bz2 *.vmdk *.ovf -mkdir dist - -# Export for XenServer -which faketime >/dev/null 2>&1 && which vhd-util >/dev/null 2>&1 -if [ $? == 0 ]; then +function remove_shares() { + log INFO "removing shared folders from appliance..." + set +e + local shared_folders=`vboxmanage showvminfo "${appliance_build_name}" | grep Name | grep Host` + if [ "${shared_folders}" == "" ]; then + return 0 + fi + folder_name=`echo "${shared_folders}" | head -1 | cut -c 8- | cut -d \' -f 1` + vboxmanage sharedfolder remove "${appliance_build_name}" --name "${folder_name}" + ! (vboxmanage showvminfo "${appliance_build_name}" | grep Name | grep Host) + local result=$? set -e - vboxmanage internalcommands converttoraw -format vdi "$hdd_path" img.raw - vhd-util convert -s 0 -t 1 -i img.raw -o stagefixed.vhd - faketime '2010-01-01' vhd-util convert -s 1 -t 2 -i stagefixed.vhd -o ${appliance_build_name}-xen.vhd - rm *.bak - bzip2 ${appliance_build_name}-xen.vhd - echo "${appliance_build_name} exported for XenServer: dist/${appliance_build_name}-xen.vhd.bz2" -else - echo "** Skipping ${appliance_build_name} export for XenServer: faketime or vhd-util command is missing. **" - echo "** faketime source code is available from https://github.com/wolfcw/libfaketime **" -fi + if [ ${result} -eq 0 ]; then + log INFO "...veewee appliance shared folders removed" + else + log INFO "...veewee appliance still has shared folders" + fi + return ${result} +} -# Exit shell if exporting fails for any format -set -e +function compact_hdd() { + log INFO "compacting image" + vboxmanage modifyhd "${1}" --compact +} -# Export for KVM -vboxmanage internalcommands converttoraw -format vdi "$hdd_path" raw.img -qemu-img convert -f raw -c -O qcow2 raw.img ${appliance_build_name}-kvm.qcow2 -rm raw.img -bzip2 ${appliance_build_name}-kvm.qcow2 -echo "${appliance_build_name} exported for KVM: dist/${appliance_build_name}-kvm.qcow2.bz2" +function xen_server_export() { + log INFO "creating xen server export" + local hdd_path="${1}" + set +e + which faketime >/dev/null 2>&1 && which vhd-util >/dev/null 2>&1 + local result=$? + set -e + if [ ${result} == 0 ]; then + vboxmanage internalcommands converttoraw -format vdi "${hdd_path}" img.raw + vhd-util convert -s 0 -t 1 -i img.raw -o stagefixed.vhd + faketime '2010-01-01' vhd-util convert -s 1 -t 2 -i stagefixed.vhd -o "${appliance_build_name}-xen.vhd" + rm *.bak + bzip2 "${appliance_build_name}-xen.vhd" + mv "${appliance_build_name}-xen.vhd.bz2" dist/ + log INFO "${appliance} exported for XenServer: dist/${appliance_build_name}-xen.vhd.bz2" + else + log WARN "** Skipping ${appliance_build_name} export for XenServer: faketime or vhd-util command is missing. **" + log WARN "** faketime source code is available from https://github.com/wolfcw/libfaketime **" + fi +} -# Export both ova and vmdk for VMWare -vboxmanage clonehd $hdd_uuid ${appliance_build_name}-vmware.vmdk --format VMDK -bzip2 ${appliance_build_name}-vmware.vmdk -echo "${appliance_build_name} exported for VMWare: dist/${appliance_build_name}-vmware.vmdk.bz2" -vboxmanage export $machine_uuid --output ${appliance_build_name}-vmware.ovf -mv ${appliance_build_name}-vmware.ovf ${appliance_build_name}-vmware.ovf-orig -java -cp convert Convert convert_ovf_vbox_to_esx.xslt ${appliance_build_name}-vmware.ovf-orig ${appliance_build_name}-vmware.ovf -tar -cf ${appliance_build_name}-vmware.ova ${appliance_build_name}-vmware.ovf ${appliance_build_name}-vmware-disk[0-9].vmdk -rm -f ${appliance_build_name}-vmware.ovf ${appliance_build_name}-vmware.ovf-orig ${appliance_build_name}-vmware-disk[0-9].vmdk -echo "${appliance_build_name} exported for VMWare: dist/${appliance_build_name}-vmware.ova" +function kvm_export() { + set +e + which faketime >/dev/null 2>&1 && which vhd-util >/dev/null 2>&1 + local result=$? + set -e + if [ ${result} == 0 ]; then + log INFO "creating kvm export" + local hdd_path="${1}" + vboxmanage internalcommands converttoraw -format vdi "${hdd_path}" raw.img + qemu-img convert -f raw -c -O qcow2 raw.img "${appliance_build_name}-kvm.qcow2" + add_on_exit rm -f raw.img + bzip2 "${appliance_build_name}-kvm.qcow2" + mv "${appliance_build_name}-kvm.qcow2.bz2" dist/ + log INFO "${appliance} exported for KVM: dist/${appliance_build_name}-kvm.qcow2.bz2" + else + log WARN "** Skipping ${appliance_build_name} export for KVM: qemu-img is missing. **" + fi +} -# Export for HyperV -vboxmanage clonehd $hdd_uuid ${appliance_build_name}-hyperv.vhd --format VHD -# HyperV doesn't support import a zipped image from S3, but we create a zipped version to save space on the jenkins box -zip ${appliance_build_name}-hyperv.vhd.zip ${appliance_build_name}-hyperv.vhd -echo "${appliance_build_name} exported for HyperV: dist/${appliance_build_name}-hyperv.vhd" +function vmware_export() { + log INFO "creating vmware export" + local machine_uuid="${1}" + local hdd_uuid="${2}" + vboxmanage clonehd "${hdd_uuid}" "${appliance_build_name}-vmware.vmdk" --format VMDK + bzip2 "${appliance_build_name}-vmware.vmdk" + mv "${appliance_build_name}-vmware.vmdk.bz2" dist/ + vboxmanage export "${machine_uuid}" --output "${appliance_build_name}-vmware.ovf" + log INFO "${appliance} exported for VMWare: dist/${appliance_build_name}-vmware.{vmdk.bz2,ovf}" + add_on_exit rm -f ${appliance_build_name}-vmware.ovf + add_on_exit rm -f ${appliance_build_name}-vmware-disk[0-9].vmdk -mv *-hyperv.vhd *-hyperv.vhd.zip *.bz2 *.ova dist/ + # xsltproc doesn't support this XSLT so we use java to run this one XSLT + mv ${appliance_build_name}-vmware.ovf ${appliance_build_name}-vmware.ovf-orig + java -cp convert Convert convert_ovf_vbox_to_esx.xslt \ + ${appliance_build_name}-vmware.ovf-orig \ + ${appliance_build_name}-vmware.ovf + add_on_exit rm -f ${appliance_build_name}-vmware.ovf-orig -rm -rf "definitions/${appliance_build_name}" + tar -cf ${appliance_build_name}-vmware.ova \ + ${appliance_build_name}-vmware.ovf \ + ${appliance_build_name}-vmware-disk[0-9].vmdk + mv ${appliance_build_name}-vmware.ova dist/ + log INFO "${appliance} exported for VMWare: dist/${appliance_build_name}-vmware.ova" +} + +function hyperv_export() { + log INFO "creating hyperv export" + local hdd_uuid="${1}" + vboxmanage clonehd "${hdd_uuid}" "${appliance_build_name}-hyperv.vhd" --format VHD + # HyperV doesn't support import a zipped image from S3, + # but we create a zipped version to save space on the jenkins box + zip "${appliance_build_name}-hyperv.vhd.zip" "${appliance_build_name}-hyperv.vhd" + mv "${appliance_build_name}-hyperv.vhd.zip" "${appliance_build_name}-hyperv.vhd" dist/ + log INFO "${appliance} exported for HyperV: dist/${appliance_build_name}-hyperv.vhd.zip" +} + +### +### Main invocation +### + +function main() { + prepare + create_definition + veewee_destroy # in case of left-over cruft from failed build + add_on_exit veewee_destroy + veewee_build + retry 10 check_appliance_shutdown + retry 10 remove_shares + + # Get appliance uuids + local vm_info=`vboxmanage showvminfo "${appliance_build_name}"` + local machine_uuid=`echo "${vm_info}" | grep UUID | head -1 | awk '{print $2}'` + local hdd_uuid=`echo "${vm_info}" | grep vdi | head -1 | awk '{print $8}' | cut -d ')' -f 1` + local hdd_path=`vboxmanage list hdds | grep "${appliance_build_name}\/" | grep vdi | \ + cut -c 14- | sed ${sed_regex_option} 's/^ *//'` + + compact_hdd "${hdd_uuid}" + xen_server_export "${hdd_path}" + kvm_export "${hdd_path}" + vmware_export "${machine_uuid}" "${hdd_uuid}" + hyperv_export "${hdd_uuid}" + log INFO "BUILD SUCCESSFUL" +} + +# we only run main() if not source-d +return 2>/dev/null || main From 35ba68444b420ac05e09f3fd06fa9d2151ebd92b Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 15:31:58 +0200 Subject: [PATCH 34/60] CLOUDSTACK-7143: add the ability to inject a custom SSH key When working on the systemvm in isolation, or using vagrant or similar tools, it can be useful to inject a custom SSH key before merging a management server systemvm.iso into it. This option allows that. It should _not_ have effect on management-server-managed vms which always get their SSH keys injected. --- tools/appliance/build.sh | 17 +++++++++-- .../systemvmtemplate/authorized_keys.sh | 29 +++++++++++++++++++ .../configure_systemvm_services.sh | 1 - .../systemvmtemplate/definition.rb | 1 + .../definitions/systemvmtemplate/zerodisk.sh | 2 +- 5 files changed, 46 insertions(+), 4 deletions(-) create mode 100644 tools/appliance/definitions/systemvmtemplate/authorized_keys.sh diff --git a/tools/appliance/build.sh b/tools/appliance/build.sh index 39bcba4bbfa..d15d2eec3a4 100755 --- a/tools/appliance/build.sh +++ b/tools/appliance/build.sh @@ -23,7 +23,7 @@ function usage() { cat < /root/.ssh/authorized_keys +chmod 600 /root/.ssh/authorized_keys diff --git a/tools/appliance/definitions/systemvmtemplate/configure_systemvm_services.sh b/tools/appliance/definitions/systemvmtemplate/configure_systemvm_services.sh index c2dcf6a26c2..b9476b6be32 100644 --- a/tools/appliance/definitions/systemvmtemplate/configure_systemvm_services.sh +++ b/tools/appliance/definitions/systemvmtemplate/configure_systemvm_services.sh @@ -68,7 +68,6 @@ configure_services() { mkdir -p /var/cache/cloud mkdir -p /usr/share/cloud mkdir -p /usr/local/cloud - mkdir -p /root/.ssh # Fix haproxy directory issue mkdir -p /var/lib/haproxy diff --git a/tools/appliance/definitions/systemvmtemplate/definition.rb b/tools/appliance/definitions/systemvmtemplate/definition.rb index c560e01c120..691952f37aa 100644 --- a/tools/appliance/definitions/systemvmtemplate/definition.rb +++ b/tools/appliance/definitions/systemvmtemplate/definition.rb @@ -86,6 +86,7 @@ config = { 'install_systemvm_packages.sh', 'configure_conntrack.sh', 'configure_systemvm_services.sh', + 'authorized_keys.sh', # cleanup & space-saving 'cleanup.sh', 'zerodisk.sh' diff --git a/tools/appliance/definitions/systemvmtemplate/zerodisk.sh b/tools/appliance/definitions/systemvmtemplate/zerodisk.sh index 64a51eace8c..581e868edc9 100644 --- a/tools/appliance/definitions/systemvmtemplate/zerodisk.sh +++ b/tools/appliance/definitions/systemvmtemplate/zerodisk.sh @@ -23,7 +23,7 @@ set -x function cleanup_veewee() { # this has to be here since it is the last file to run (and we remove ourselves) rm -fv /root/*.iso - rm -fv /root/{apt_upgrade,build_time,cleanup,install_systemvm_packages,zerodisk}.sh + rm -fv /root/{apt_upgrade,authorized_keys,build_time,cleanup,install_systemvm_packages,zerodisk}.sh rm -fv /root/configure_{acpid,conntrack,grub,locale,login,networking,systemvm_services}.sh rm -fv .veewee_version .veewee_params .vbox_version } From 33fd6894a0ecf6133a2271e44f55d61687d67b2d Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 15:41:16 +0200 Subject: [PATCH 35/60] CLOUDSTACK-7143: use shar to inject cloud-scripts from working copy The current build downloads its script from master by fetching a cloudstack tarball. Besides being an unneeded load on the apache git server, this is a problem when working on a branch and wanting to inject a different set of scripts. It also makes it pretty likely that the injected copy of the script will not match what a production release wants, so there is very little chance of not needing to overwrite the scripts. Ideally we would just rsync over some files. However, veewee does not provide an option to do that. In order to keep a 'cleanly veewee-only' build possible, and work with any recent veewee version, in this change we restor to using shar (http://en.wikipedia.org/wiki/Shar) to produce an archive which can execute as a script, which we feed to veewee to execute. --- tools/appliance/build.sh | 3 ++ .../configure_systemvm_services.sh | 24 +++------ .../systemvmtemplate/definition.rb | 1 + tools/appliance/shar_cloud_scripts.sh | 53 +++++++++++++++++++ 4 files changed, 65 insertions(+), 16 deletions(-) create mode 100755 tools/appliance/shar_cloud_scripts.sh diff --git a/tools/appliance/build.sh b/tools/appliance/build.sh index d15d2eec3a4..84d9648a883 100755 --- a/tools/appliance/build.sh +++ b/tools/appliance/build.sh @@ -244,6 +244,9 @@ function create_definition() { set -e add_on_exit rm -rf "definitions/${appliance_build_name}" fi + + ./shar_cloud_scripts.sh + add_on_exit rm -f cloud_scripts_shar_archive.sh } function prepare() { diff --git a/tools/appliance/definitions/systemvmtemplate/configure_systemvm_services.sh b/tools/appliance/definitions/systemvmtemplate/configure_systemvm_services.sh index b9476b6be32..e682fc729db 100644 --- a/tools/appliance/definitions/systemvmtemplate/configure_systemvm_services.sh +++ b/tools/appliance/definitions/systemvmtemplate/configure_systemvm_services.sh @@ -31,21 +31,13 @@ function configure_apache2() { } function install_cloud_scripts() { - # Get config files from master - snapshot_url="https://git-wip-us.apache.org/repos/asf?p=cloudstack.git;a=snapshot;h=HEAD;sf=tgz" - snapshot_dir="/opt/cloudstack*" - cd /opt - wget --no-check-certificate $snapshot_url -O cloudstack.tar.gz - tar -zxvf cloudstack.tar.gz --wildcards 'cloudstack-HEAD-???????/systemvm' - cp -rv $snapshot_dir/systemvm/patches/debian/config/* / - cp -rv $snapshot_dir/systemvm/patches/debian/vpn/* / - mkdir -p /usr/share/cloud/ - cd $snapshot_dir/systemvm/patches/debian/config - tar -cvf /usr/share/cloud/cloud-scripts.tar * - cd $snapshot_dir/systemvm/patches/debian/vpn - tar -rvf /usr/share/cloud/cloud-scripts.tar * - cd /opt - rm -fr $snapshot_dir cloudstack.tar.gz + # ./cloud_scripts/ has been put there by ../../cloud_scripts_shar_archive.sh + rsync -av ./cloud_scripts/ / + chmod +x /opt/cloud/bin/* \ + /root/{clearUsageRules.sh,reconfigLB.sh,monitorServices.py} \ + /etc/init.d/{cloud,cloud-early-config,cloud-passwd-srvr,postinit} \ + /etc/cron.daily/cloud-cleanup \ + /etc/profile.d/cloud.sh chkconfig --add cloud-early-config chkconfig cloud-early-config on @@ -73,6 +65,7 @@ configure_services() { mkdir -p /var/lib/haproxy install_cloud_scripts + do_signature chkconfig xl2tpd off @@ -84,7 +77,6 @@ configure_services() { chkconfig radvd off configure_apache2 - do_signature } return 2>/dev/null || configure_services diff --git a/tools/appliance/definitions/systemvmtemplate/definition.rb b/tools/appliance/definitions/systemvmtemplate/definition.rb index 691952f37aa..59654065617 100644 --- a/tools/appliance/definitions/systemvmtemplate/definition.rb +++ b/tools/appliance/definitions/systemvmtemplate/definition.rb @@ -85,6 +85,7 @@ config = { # turning it into a systemvm 'install_systemvm_packages.sh', 'configure_conntrack.sh', + '../../cloud_scripts_shar_archive.sh', 'configure_systemvm_services.sh', 'authorized_keys.sh', # cleanup & space-saving diff --git a/tools/appliance/shar_cloud_scripts.sh b/tools/appliance/shar_cloud_scripts.sh new file mode 100755 index 00000000000..33268e85248 --- /dev/null +++ b/tools/appliance/shar_cloud_scripts.sh @@ -0,0 +1,53 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# since veewee wants .sh files to execute, we'll give it a shar + +set -e +set -x + +# where we are running this script from +CURR_DIR=${PWD} +# where this script is +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +# where cloudstack is checked out +cd ${SCRIPT_DIR}/../.. +CLOUDSTACK_DIR=${PWD} +cd ${CURR_DIR} +# ensure we are running in isolation +TEMP_DIR=`mktemp -d -t shar_cloud` + +cd ${TEMP_DIR} +mkdir cloud_scripts +mkdir -p cloud_scripts/opt/cloudstack +cp -r ${CLOUDSTACK_DIR}/systemvm/patches/debian/config/* cloud_scripts/ +cp -r ${CLOUDSTACK_DIR}/systemvm/patches/debian/vpn/* cloud_scripts/ + +mkdir -p cloud_scripts/usr/share/cloud +cd ${CLOUDSTACK_DIR}/systemvm/patches/debian/config +tar -cf ${TEMP_DIR}/cloud_scripts/usr/share/cloud/cloud-scripts.tar * +cd ${CLOUDSTACK_DIR}/systemvm/patches/debian/vpn +tar -rf ${TEMP_DIR}/cloud_scripts/usr/share/cloud/cloud-scripts.tar * + +cd ${TEMP_DIR} +shar `find . -print` > ${CURR_DIR}/cloud_scripts_shar_archive.sh + +cd ${CURR_DIR} +rm -rf ${TEMP_DIR} +chmod +x cloud_scripts_shar_archive.sh +echo cloud_scripts are in cloud_scripts_shar_archive.sh From dd34d101596b06355182bd22218efdb9e3e3b659 Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 21 Jul 2014 15:59:47 +0200 Subject: [PATCH 36/60] CLOUDSTACK-7143: add tests for the appliance build Create a new minimal 'debianbase' definition which is a veewee template that's a lot like the systemvmtemplate, but does not have any systemvm-ness in it. Use it to create a new test.sh which tests a few common invocations of build.sh work as desired. This is mainly useful for debugging whether the appliance build process is working / consistent; in order to test a systemvm itself it should really first be merged with systemvm.iso. --- tools/appliance/build.sh | 15 +- .../definitions/debianbase/cleanup.sh | 49 +++++ .../definitions/debianbase/configure_login.sh | 78 ++++++++ .../definitions/debianbase/definition.rb | 93 +++++++++ .../definitions/debianbase/preseed.cfg | 129 +++++++++++++ .../definitions/debianbase/zerodisk.sh | 43 +++++ tools/appliance/test.sh | 180 ++++++++++++++++++ 7 files changed, 581 insertions(+), 6 deletions(-) create mode 100644 tools/appliance/definitions/debianbase/cleanup.sh create mode 100644 tools/appliance/definitions/debianbase/configure_login.sh create mode 100644 tools/appliance/definitions/debianbase/definition.rb create mode 100644 tools/appliance/definitions/debianbase/preseed.cfg create mode 100644 tools/appliance/definitions/debianbase/zerodisk.sh create mode 100644 tools/appliance/test.sh diff --git a/tools/appliance/build.sh b/tools/appliance/build.sh index 84d9648a883..90a1ad95164 100755 --- a/tools/appliance/build.sh +++ b/tools/appliance/build.sh @@ -107,12 +107,7 @@ BUILD_NUMBER="${4:-${BUILD_NUMBER:-}}" # (debian) os architecture to build arch="${5:-${arch:-i386}}" -if [ "${appliance}" == "systemvm64template" ]; then - arch="amd64" - export VM_ARCH="${arch}" - rm -rf definitions/systemvm64template - cp -r definitions/systemvmtemplate definitions/systemvm64template -fi +export VM_ARCH="${arch}" # optional root SSH public key to write to /root/.ssh/authorized_keys # note the cs management server overwrites this, so the only reason to @@ -231,6 +226,14 @@ function retry() { ### function create_definition() { + if [ "${appliance}" == "systemvm64template" ]; then + arch="amd64" + export VM_ARCH="${arch}" + rm -rf definitions/systemvm64template # in case of left-over cruft from failed build + cp -r definitions/systemvmtemplate definitions/systemvm64template + add_on_exit rm -rf definitions/systemvm64template + fi + if [ "${appliance}" != "${appliance_build_name}" ]; then cp -r "definitions/${appliance}" "definitions/${appliance_build_name}" set +e diff --git a/tools/appliance/definitions/debianbase/cleanup.sh b/tools/appliance/definitions/debianbase/cleanup.sh new file mode 100644 index 00000000000..cd32d5b42aa --- /dev/null +++ b/tools/appliance/definitions/debianbase/cleanup.sh @@ -0,0 +1,49 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e +set -x + +function cleanup_apt() { + #apt-get -y remove linux-headers-$(uname -r) build-essential + apt-get -y remove dictionaries-common busybox + apt-get -y autoremove + apt-get autoclean + apt-get clean +} + +# Removing leftover leases and persistent rules +function cleanup_dhcp() { + rm -f /var/lib/dhcp/* +} + +# Make sure Udev doesn't block our network +function cleanup_dev() { + echo "cleaning up udev rules" + rm -f /etc/udev/rules.d/70-persistent-net.rules + rm -rf /dev/.udev/ + rm -f /lib/udev/rules.d/75-persistent-net-generator.rules +} + +function cleanup() { + cleanup_apt + cleanup_dhcp + cleanup_dev +} + +return 2>/dev/null || cleanup diff --git a/tools/appliance/definitions/debianbase/configure_login.sh b/tools/appliance/definitions/debianbase/configure_login.sh new file mode 100644 index 00000000000..36fccabc5fa --- /dev/null +++ b/tools/appliance/definitions/debianbase/configure_login.sh @@ -0,0 +1,78 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e +set -x + +function add_admin_group() { + groupadd -f -r admin +} + +function configure_cloud_user() { + usermod -a -G admin cloud + mkdir -p /home/cloud/.ssh + chmod 700 /home/cloud/.ssh + echo "cloud:`openssl rand -base64 32`" | chpasswd +} + +function configure_sudoers() { + cat >/etc/sudoers < /etc/sudoers.d/cloud +} + +# sshd_config is overwritten from cloud_scripts +#function configure_sshd() { +# grep "UseDNS no" /etc/ssh/sshd_config && \ +# grep "PasswordAuthentication no" /etc/ssh/sshd_config && \ +# return +# # Tweak sshd to prevent DNS resolution (speed up logins) +# echo 'UseDNS no' >> /etc/ssh/sshd_config +# +# # Require ssh keys for login +# sed -i -e 's/^.*PasswordAuthentication .*$/PasswordAuthentication no/g' /etc/ssh/sshd_config +#} + +function configure_inittab() { + grep "vc:2345:respawn:/sbin/getty" /etc/inittab && return + + # Fix inittab + cat >> /etc/inittab << EOF + +vc:2345:respawn:/sbin/getty 38400 hvc0 +EOF +} + +function configure_login() { + add_admin_group + configure_cloud_user + configure_sudoers + # configure_sshd + configure_inittab +} + +return 2>/dev/null || configure_login diff --git a/tools/appliance/definitions/debianbase/definition.rb b/tools/appliance/definitions/debianbase/definition.rb new file mode 100644 index 00000000000..f430f33548f --- /dev/null +++ b/tools/appliance/definitions/debianbase/definition.rb @@ -0,0 +1,93 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +arch = ENV['VM_ARCH'] || 'i386' + +architectures = { + :i386 => { + :os_type_id => 'Debian', + :iso_file => 'debian-7.6.0-i386-netinst.iso', + :iso_src => 'http://cdimage.debian.org/debian-cd/7.6.0/i386/iso-cd/debian-7.6.0-i386-netinst.iso', + :iso_md5 => '528e1a7315da1bbf50bd4d187880a519', + }, + :amd64 => { + :os_type_id => 'Debian_64', + :iso_file => 'debian-7.6.0-amd64-netinst.iso', + :iso_src => 'http://cdimage.debian.org/debian-cd/7.6.0/amd64/iso-cd/debian-7.6.0-amd64-netinst.iso', + :iso_md5 => '8a3c2ad7fd7a9c4c7e9bcb5cae38c135' + } +} + +config = { + :cpu_count => '1', + :memory_size => '256', + :disk_size => '2500', :disk_format => 'VDI', :hostiocache => 'off', + :iso_download_timeout => '1200', + :boot_wait => '10', + :boot_cmd_sequence => [ + '', + 'install ', + 'preseed/url=http://%IP%:%PORT%/preseed.cfg ', + 'debian-installer=en_US ', + 'auto ', + 'locale=en_US ', + 'kbd-chooser/method=us ', + 'netcfg/get_hostname=systemvm ', + 'netcfg/get_domain=apache.org ', + 'fb=false ', + 'debconf/frontend=noninteractive ', + 'console-setup/ask_detect=false ', + 'console-keymaps-at/keymap=us ', + 'keyboard-configuration/xkb-keymap=us ', + '' + ], + :kickstart_port => '7122', + :kickstart_timeout => '1200', + :kickstart_file => 'preseed.cfg', + :ssh_login_timeout => '1200', + :ssh_user => 'root', + :ssh_password => 'password', + :ssh_key => '', + :ssh_host_port => '7222', + :ssh_guest_port => '22', + :sudo_cmd => "echo '%p'|sudo -S sh '%f'", + :shutdown_cmd => 'halt -p', + :postinstall_files => [ + # basic minimal vm creation + 'build_time.sh', + 'apt_upgrade.sh', + 'configure_grub.sh', + 'configure_locale.sh', + 'configure_login.sh', + 'configure_networking.sh', + 'configure_acpid.sh', + # turning it into a systemvm + 'install_systemvm_packages.sh', + 'configure_conntrack.sh', + '../../cloud_scripts_shar_archive.sh', + 'configure_systemvm_services.sh', + 'authorized_keys.sh', + # cleanup & space-saving + 'cleanup.sh', + 'zerodisk.sh' + ], + :postinstall_timeout => '1200' +} + +config.merge! architectures[arch.to_sym] + +Veewee::Definition.declare(config) diff --git a/tools/appliance/definitions/debianbase/preseed.cfg b/tools/appliance/definitions/debianbase/preseed.cfg new file mode 100644 index 00000000000..e6e5adbdd32 --- /dev/null +++ b/tools/appliance/definitions/debianbase/preseed.cfg @@ -0,0 +1,129 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +### Localization +# Locale sets language and country. +d-i debian-installer/locale string en_US + +# Keyboard selection. +d-i console-keymaps-at/keymap select us + +### Network configuration +d-i netcfg/choose_interface select auto +d-i netcfg/get_hostname string debianbase +d-i netcfg/get_domain string cloudstack.org + +### Mirror settings +d-i mirror/country string manual +d-i mirror/http/hostname string http.us.debian.org +d-i mirror/http/directory string /debian +d-i mirror/http/proxy string + +### Clock and time zone setup +d-i clock-setup/utc boolean true +d-i time/zone string UTC +d-i clock-setup/ntp boolean true + +### Partitioning +d-i partman-auto/disk string /dev/sda +d-i partman-auto/method string regular +d-i partman-auto/choose_recipe select atomic +d-i partman-auto/expert_recipe string \ + boot-root :: \ + 30 50 100 ext4 \ + $primary{ } $bootable{ } \ + method{ format } format{ } \ + use_filesystem{ } filesystem{ ext4 } \ + mountpoint{ /boot } \ + . \ + 300 40 400 ext4 \ + method{ format } format{ } \ + use_filesystem{ } filesystem{ ext4 } \ + mountpoint{ / } \ + . \ + 50 100 200 ext4 \ + method{ format } format{ } \ + use_filesystem{ } filesystem{ ext4 } \ + mountpoint{ /home } \ + . \ + 650 20 1100 ext4 \ + method{ format } format{ } \ + use_filesystem{ } filesystem{ ext4 } \ + mountpoint{ /usr } \ + . \ + 400 40 500 ext4 \ + method{ format } format{ } \ + use_filesystem{ } filesystem{ ext4 } \ + mountpoint{ /opt } \ + . \ + 450 60 1000 ext4 \ + method{ format } format{ } \ + use_filesystem{ } filesystem{ ext4 } \ + mountpoint{ /var } \ + . \ + 50 70 400 ext4 \ + method{ format } format{ } \ + use_filesystem{ } filesystem{ ext4 } \ + mountpoint{ /tmp } \ + . \ + 70 512 300% linux-swap \ + method{ swap } format{ } \ + . +d-i partman/confirm_write_new_label boolean true +d-i partman/choose_partition select finish +d-i partman/confirm boolean true +d-i partman/confirm_nooverwrite boolean true + +### Base system installation +# ... + +### Account setup +d-i passwd/root-login boolean true +d-i passwd/root-password password password +d-i passwd/root-password-again password password +d-i passwd/user-fullname string Cloud Stack +d-i passwd/username string cloud +d-i passwd/user-password password cloud +d-i passwd/user-password-again password cloud +d-i user-setup/encrypt-home boolean false +d-i user-setup/allow-password-weak boolean true +d-i passwd/user-default-groups string audio cdrom video admin + +### Apt setup +# ... + +### Package selection +tasksel tasksel/first multiselect ssh-server +d-i pkgsel/include string openssh-server ntp acpid sudo bzip2 +# Allowed values: none, safe-upgrade, full-upgrade +d-i pkgsel/upgrade select none + +popularity-contest popularity-contest/participate boolean false + +### Boot loader installation +d-i grub-installer/only_debian boolean true +d-i finish-install/reboot_in_progress note + +### Preseeding other packages +libssl1.0.0 libssl1.0.0/restart-services string +libssl1.0.0 libssl1.0.0/restart-failed error + +#### Advanced options +# Prevent packaged version of VirtualBox Guest Additions being installed: +d-i preseed/early_command string sed -i \ + '/in-target/idiscover(){/sbin/discover|grep -v VirtualBox;}' \ + /usr/lib/pre-pkgsel.d/20install-hwpackages diff --git a/tools/appliance/definitions/debianbase/zerodisk.sh b/tools/appliance/definitions/debianbase/zerodisk.sh new file mode 100644 index 00000000000..581e868edc9 --- /dev/null +++ b/tools/appliance/definitions/debianbase/zerodisk.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e +set -x + +# clean up stuff copied in by veewee +function cleanup_veewee() { + # this has to be here since it is the last file to run (and we remove ourselves) + rm -fv /root/*.iso + rm -fv /root/{apt_upgrade,authorized_keys,build_time,cleanup,install_systemvm_packages,zerodisk}.sh + rm -fv /root/configure_{acpid,conntrack,grub,locale,login,networking,systemvm_services}.sh + rm -fv .veewee_version .veewee_params .vbox_version +} + +# Zero out the free space to save space in the final image: +function zero_disk() { + cleanup_veewee + + for path in / /boot /usr /var /opt /tmp /home + do + dd if=/dev/zero of=${path}/zero bs=1M || true + sync + rm -f ${path}/zero + done +} + +return 2>/dev/null || zero_disk diff --git a/tools/appliance/test.sh b/tools/appliance/test.sh new file mode 100644 index 00000000000..5cd7316eac3 --- /dev/null +++ b/tools/appliance/test.sh @@ -0,0 +1,180 @@ +#!/bin/bash -xl +# note: the -l is needed here for bash to always make a login shell and load rvm if it hasn't been loaded +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# test script for build.sh which tries a variety of options/configs to make different vms + +set -e + +DEBUG="${DEBUG:-}" +TRACE="${TRACE:-0}" + +### +### Configuration +### + +if [[ "${DEBUG}" == "1" ]]; then + set -x +fi + +# which test to run +test_to_run=${1:-} +# build.sh settings for running the tests +appliance=debianbase +version=`date "+%Y%m%d%H%M%S"` +branch=`git status | grep '# On branch' | awk '{print $4}'` +BUILD_NUMBER="${BUILD_NUMBER:-}" +ssh_key= + +# where we are running the tests from +CURR_DIR=${PWD} +# where this script is +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +# ensure we are running in isolation +if [ ${CURR_DIR} == ${SCRIPT_DIR} ]; then + mkdir -p ../appliance-work + cd ../appliance-work + CURR_DIR=${PWD} +fi + +### +### testing 'framework' +### + +function test_result() { + log INFO "$@" + add_on_exit log INFO "$@" +} + +function run_test() { + set +e + cleanup + fixture + log INFO running test: "$@" + eval $@ + result=$? + if ${result}; then + test_result "$@" FAIL + else + test_result "$@" OK + fi + cleanup + set -e +} + +function cleanup() { + ( + cd ${CURR_DIR}; + rm -rf iso definitions Gemfile shar_cloud_scripts convert_ovf_vbox_to_esx.xslt .rvmrc; + ) +} + +function fixture() { + ( + cd ${CURR_DIR}; + mkdir -p ${SCRIPT_DIR}/iso; + ln -s ${SCRIPT_DIR}/iso; + mkdir definitions; + ln -s ${SCRIPT_DIR}/definitions/${appliance} definitions/${appliance}; + + ln -s ${SCRIPT_DIR}/Gemfile; + ln -s ${SCRIPT_DIR}/shar_cloud_scripts.sh; + ln -s ${SCRIPT_DIR}/convert_ovf_vbox_to_esx.xslt; + ln -s ${SCRIPT_DIR}/.rvmrc; + ) +} + +### +### Test definitions +### + +function do_test_vm() { + prepare + create_definition + veewee_build + retry 10 check_appliance_shutdown + retry 10 remove_shares + veewee_destroy +} + +function do_test_export() { + prepare + create_definition + veewee_build + retry 10 check_appliance_shutdown + retry 10 remove_shares + + # Get appliance uuids + local vm_info=`vboxmanage showvminfo "${appliance_build_name}"` + local machine_uuid=`echo "${vm_info}" | grep UUID | head -1 | awk '{print $2}'` + local hdd_uuid=`echo "${vm_info}" | grep vdi | head -1 | awk '{print $8}' | cut -d ')' -f 1` + local hdd_path=`vboxmanage list hdds | grep "${appliance_build_name}\/" | grep vdi | \ + cut -c 14- | sed ${sed_regex_option} 's/^ *//'` + + compact_hdd "${hdd_uuid}" + xen_server_export "${hdd_path}" + kvm_export "${hdd_path}" + vmware_export "${machine_uuid}" "${hdd_uuid}" + hyperv_export "${hdd_uuid}" + + veewee_destroy +} + +function test_basic_veewee_invocation() { + appliance=debianbase + appliance_build_name=${appliance}${branch_tag}${version_tag} + do_test_vm +} + +function test_export() { + appliance=debianbase + appliance_build_name=${appliance}${branch_tag}${version_tag} + do_test_export +} + +function test_systemvm() { + appliance=systemvmtemplate + appliance_build_name=${appliance}${branch_tag}${version_tag} + do_test_vm +} + +function test_systemvm64() { + appliance=systemvm64template + appliance_build_name=${appliance}${branch_tag}${version_tag} + do_test_vm +} + +function test_suite() { + if [ "${test_to_run}" == "" ]; then + # list of all tests goes here + run_test test_basic_veewee_invocation + run_test test_systemvm + run_test test_systemvm64 + run_test test_export + else + run_test "${test_to_run}" + fi +} + +### +### Main invocation +### + +source ${SCRIPT_DIR}/build.sh +return 2>/dev/null || test_suite From 634560e38193e06c8fb80459fcf117a5649b2703 Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Tue, 22 Jul 2014 13:11:55 +0200 Subject: [PATCH 37/60] CLOUDSTACK-7143: fix a bug in the debianbase template that was referencing deleted scripts --- .../appliance/definitions/debianbase/definition.rb | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/tools/appliance/definitions/debianbase/definition.rb b/tools/appliance/definitions/debianbase/definition.rb index f430f33548f..9e41e65946d 100644 --- a/tools/appliance/definitions/debianbase/definition.rb +++ b/tools/appliance/definitions/debianbase/definition.rb @@ -68,20 +68,7 @@ config = { :shutdown_cmd => 'halt -p', :postinstall_files => [ # basic minimal vm creation - 'build_time.sh', - 'apt_upgrade.sh', - 'configure_grub.sh', - 'configure_locale.sh', 'configure_login.sh', - 'configure_networking.sh', - 'configure_acpid.sh', - # turning it into a systemvm - 'install_systemvm_packages.sh', - 'configure_conntrack.sh', - '../../cloud_scripts_shar_archive.sh', - 'configure_systemvm_services.sh', - 'authorized_keys.sh', - # cleanup & space-saving 'cleanup.sh', 'zerodisk.sh' ], From a7d07bf3dfd82260ab8f73e06d34c3802f4eec76 Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Tue, 22 Jul 2014 13:12:49 +0200 Subject: [PATCH 38/60] CLOUDSTACK-7143: fix a bug where a script file rename wasn't propagated to build.sh --- tools/appliance/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/appliance/build.sh b/tools/appliance/build.sh index 90a1ad95164..9ae83f192ab 100755 --- a/tools/appliance/build.sh +++ b/tools/appliance/build.sh @@ -238,7 +238,7 @@ function create_definition() { cp -r "definitions/${appliance}" "definitions/${appliance_build_name}" set +e sed ${sed_regex_option} -i -e "s/^CLOUDSTACK_RELEASE=.+/CLOUDSTACK_RELEASE=${version}/" \ - "definitions/${appliance_build_name}/postinstall.sh" + "definitions/${appliance_build_name}/configure_systemvm_services.sh" if [ ! -z "${ssh_key}" ]; then # ssh key lines can contain / sed ${sed_regex_option} -i -e "s|^key=.+|key=\"${ssh_key}\"|" \ From a56392b5e4769d1b3e8cce0a8a1da89e11a24c46 Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Tue, 22 Jul 2014 13:13:54 +0200 Subject: [PATCH 39/60] CLOUDSTACK-7143: attempt at a reliable way to set up rvm/bundler on jenkins --- tools/appliance/build.sh | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/tools/appliance/build.sh b/tools/appliance/build.sh index 9ae83f192ab..282472d525e 100755 --- a/tools/appliance/build.sh +++ b/tools/appliance/build.sh @@ -252,9 +252,22 @@ function create_definition() { add_on_exit rm -f cloud_scripts_shar_archive.sh } +function setup_ruby() { + local bundle_args= + if [[ ! -z "${JENKINS_HOME}" ]]; then + # inspired by https://github.com/CloudBees-community/rubyci-clickstart/blob/master/bin/run-ci + # also see https://rvm.io/integration/jenkins + # .rvmrc won't get trusted/auto-loaded by jenkins by default + export VAGRANT_HOME=$HOME/.vagrant.d-release-cloudstack + rvm use ruby-1.9.3@vagrant-release-cloudstack --create + bundle_args="--deployment" + fi + bundle check || bundle install ${bundle_args} +} + function prepare() { log INFO "preparing for build" - bundle + setup_ruby rm -rf dist *.ova *.vhd *.vdi *.qcow* *.bz2 *.vmdk *.ovf mkdir dist } From 631a3c8722b375299b976b4b0df326d67ee30521 Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Tue, 22 Jul 2014 13:49:35 +0200 Subject: [PATCH 40/60] CLOUDSTACK-7143: install openssl early enough to allow generating passwords --- .../definitions/systemvmtemplate/install_systemvm_packages.sh | 2 +- tools/appliance/definitions/systemvmtemplate/preseed.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/appliance/definitions/systemvmtemplate/install_systemvm_packages.sh b/tools/appliance/definitions/systemvmtemplate/install_systemvm_packages.sh index d0436b43db3..dac727c4a8d 100644 --- a/tools/appliance/definitions/systemvmtemplate/install_systemvm_packages.sh +++ b/tools/appliance/definitions/systemvmtemplate/install_systemvm_packages.sh @@ -52,7 +52,7 @@ function install_packages() { ${apt_install} \ rsyslog logrotate cron chkconfig insserv net-tools ifupdown vim-tiny netbase iptables \ - openssh-server openssl e2fsprogs dhcp3-client tcpdump socat wget \ + openssh-server e2fsprogs dhcp3-client tcpdump socat wget \ python bzip2 sed gawk diffutils grep gzip less tar telnet ftp rsync traceroute psmisc lsof procps \ inetutils-ping iputils-arping httping \ dnsutils zip unzip ethtool uuid file iproute acpid virt-what sudo \ diff --git a/tools/appliance/definitions/systemvmtemplate/preseed.cfg b/tools/appliance/definitions/systemvmtemplate/preseed.cfg index cac9fc7dbc9..438488bb178 100644 --- a/tools/appliance/definitions/systemvmtemplate/preseed.cfg +++ b/tools/appliance/definitions/systemvmtemplate/preseed.cfg @@ -108,7 +108,7 @@ d-i passwd/user-default-groups string audio cdrom video admin ### Package selection tasksel tasksel/first multiselect ssh-server -d-i pkgsel/include string openssh-server ntp acpid sudo bzip2 +d-i pkgsel/include string openssh-server ntp acpid sudo bzip2 openssl # Allowed values: none, safe-upgrade, full-upgrade d-i pkgsel/upgrade select none From 41ddb6b768188676dab8102063088a1d67414e75 Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Tue, 22 Jul 2014 13:49:52 +0200 Subject: [PATCH 41/60] CLOUDSTACK-7143: make script compatible with linux mktemp --- tools/appliance/shar_cloud_scripts.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/appliance/shar_cloud_scripts.sh b/tools/appliance/shar_cloud_scripts.sh index 33268e85248..5504d16e3ae 100755 --- a/tools/appliance/shar_cloud_scripts.sh +++ b/tools/appliance/shar_cloud_scripts.sh @@ -30,7 +30,9 @@ cd ${SCRIPT_DIR}/../.. CLOUDSTACK_DIR=${PWD} cd ${CURR_DIR} # ensure we are running in isolation -TEMP_DIR=`mktemp -d -t shar_cloud` +TMPDIR=${TMPDIR:-/tmp} +TMPDIR=${TMPDIR%/} +TEMP_DIR=`mktemp -d ${TMPDIR}/shar_cloud.XXXXXXXX` cd ${TEMP_DIR} mkdir cloud_scripts From b4fceef40783f41ef5dd7a78d25dbb152515c8fd Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Tue, 22 Jul 2014 14:09:19 +0200 Subject: [PATCH 42/60] CLOUDSTACK-7143: sometimes the consolidated 'apt' command runs out of space --- tools/appliance/definitions/systemvmtemplate/definition.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/appliance/definitions/systemvmtemplate/definition.rb b/tools/appliance/definitions/systemvmtemplate/definition.rb index 59654065617..b7db768d645 100644 --- a/tools/appliance/definitions/systemvmtemplate/definition.rb +++ b/tools/appliance/definitions/systemvmtemplate/definition.rb @@ -42,7 +42,7 @@ architectures = { config = { :cpu_count => '1', :memory_size => '256', - :disk_size => '2500', :disk_format => 'VDI', :hostiocache => 'off', + :disk_size => '3000', :disk_format => 'VDI', :hostiocache => 'off', :iso_download_timeout => '1200', :boot_wait => '10', :boot_cmd_sequence => [ From 0570d371067ad22c9ff0d51f7d4526bcaa390e11 Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Tue, 22 Jul 2014 15:10:23 +0200 Subject: [PATCH 43/60] CLOUDSTACK-7143: initial apt-upgrade still running out of space (upgrading the kernel) I wonder if this is due to the 7.4.0->7.6.0 needing more space. Should see if there's more stuff that can be cleaned earlier? --- tools/appliance/definitions/systemvmtemplate/definition.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/appliance/definitions/systemvmtemplate/definition.rb b/tools/appliance/definitions/systemvmtemplate/definition.rb index b7db768d645..351a40d3449 100644 --- a/tools/appliance/definitions/systemvmtemplate/definition.rb +++ b/tools/appliance/definitions/systemvmtemplate/definition.rb @@ -42,7 +42,7 @@ architectures = { config = { :cpu_count => '1', :memory_size => '256', - :disk_size => '3000', :disk_format => 'VDI', :hostiocache => 'off', + :disk_size => '3500', :disk_format => 'VDI', :hostiocache => 'off', :iso_download_timeout => '1200', :boot_wait => '10', :boot_cmd_sequence => [ From 25bb682cd194ac39f8bca4cc99b4488c36463cec Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Tue, 22 Jul 2014 15:40:31 +0200 Subject: [PATCH 44/60] CLOUDSTACK-7143: fix the fix for rvm/ruby/bundler setup on jenkins The recommended use of --deployment goes along with the requirement to check Gemfile.lock into source control, which I don't want to do. --- tools/appliance/build.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/appliance/build.sh b/tools/appliance/build.sh index 282472d525e..d315b43e20e 100755 --- a/tools/appliance/build.sh +++ b/tools/appliance/build.sh @@ -260,7 +260,8 @@ function setup_ruby() { # .rvmrc won't get trusted/auto-loaded by jenkins by default export VAGRANT_HOME=$HOME/.vagrant.d-release-cloudstack rvm use ruby-1.9.3@vagrant-release-cloudstack --create - bundle_args="--deployment" + # do not use --deployment since that requires Gemfile.lock...and we prefer an up-to-date veewee + bundle_args="--path vendor/bundle" fi bundle check || bundle install ${bundle_args} } From d5a4a2521bc7bb6ec1450329a525d2cd9810a345 Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Tue, 22 Jul 2014 15:41:32 +0200 Subject: [PATCH 45/60] CLOUDSTACK-7143: fix do_signature to account for change to install_cloud_scripts The do_signature script required a lingering cloud-scripts.tar that was no longer there after the git export tarball fiddling magic was gone. --- .../systemvmtemplate/configure_systemvm_services.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/appliance/definitions/systemvmtemplate/configure_systemvm_services.sh b/tools/appliance/definitions/systemvmtemplate/configure_systemvm_services.sh index e682fc729db..c076bb50480 100644 --- a/tools/appliance/definitions/systemvmtemplate/configure_systemvm_services.sh +++ b/tools/appliance/definitions/systemvmtemplate/configure_systemvm_services.sh @@ -47,14 +47,14 @@ function install_cloud_scripts() { chkconfig cloud off } -do_signature() { - mkdir -p /var/cache/cloud/ - gzip -c /usr/share/cloud/cloud-scripts.tar > /usr/share/cloud/cloud-scripts.tgz +function do_signature() { + mkdir -p /var/cache/cloud/ /usr/share/cloud/ + (cd ./cloud_scripts/; tar -cvf - * | gzip > /usr/share/cloud/cloud-scripts.tgz) md5sum /usr/share/cloud/cloud-scripts.tgz | awk '{print $1}' > /var/cache/cloud/cloud-scripts-signature echo "Cloudstack Release $CLOUDSTACK_RELEASE $(date)" > /etc/cloudstack-release } -configure_services() { +function configure_services() { mkdir -p /var/www/html mkdir -p /opt/cloud/bin mkdir -p /var/cache/cloud From 91ffedb39cbeb1ab1cea96ad085153bf65947129 Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Tue, 22 Jul 2014 17:15:42 +0200 Subject: [PATCH 46/60] CLOUDSTACK-7143: a better fix for kernel upgrade disk space problems Increase / and /boot a little bit, to the point that there is enough space on them to do a kernel upgrade. With this I think we can decrease total disk space usage back down to 2.5GB. Docs for the preseed config found at http://anonscm.debian.org/gitweb/?p=d-i/debian-installer.git;a=blob_plain;f=doc/devel/partman-auto-recipe.txt;hb=HEAD --- tools/appliance/definitions/systemvmtemplate/definition.rb | 2 +- tools/appliance/definitions/systemvmtemplate/preseed.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/appliance/definitions/systemvmtemplate/definition.rb b/tools/appliance/definitions/systemvmtemplate/definition.rb index 351a40d3449..59654065617 100644 --- a/tools/appliance/definitions/systemvmtemplate/definition.rb +++ b/tools/appliance/definitions/systemvmtemplate/definition.rb @@ -42,7 +42,7 @@ architectures = { config = { :cpu_count => '1', :memory_size => '256', - :disk_size => '3500', :disk_format => 'VDI', :hostiocache => 'off', + :disk_size => '2500', :disk_format => 'VDI', :hostiocache => 'off', :iso_download_timeout => '1200', :boot_wait => '10', :boot_cmd_sequence => [ diff --git a/tools/appliance/definitions/systemvmtemplate/preseed.cfg b/tools/appliance/definitions/systemvmtemplate/preseed.cfg index 438488bb178..0d2f70b3d0b 100644 --- a/tools/appliance/definitions/systemvmtemplate/preseed.cfg +++ b/tools/appliance/definitions/systemvmtemplate/preseed.cfg @@ -50,7 +50,7 @@ d-i partman-auto/expert_recipe string \ use_filesystem{ } filesystem{ ext4 } \ mountpoint{ /boot } \ . \ - 250 40 400 ext4 \ + 400 40 500 ext4 \ method{ format } format{ } \ use_filesystem{ } filesystem{ ext4 } \ mountpoint{ / } \ From 1821f4ec06e8a8e0c57c9792049e3c8d08ff3ab3 Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Tue, 22 Jul 2014 17:20:07 +0200 Subject: [PATCH 47/60] CLOUDSTACK-7143: fix a missing apt-get update In 8e2d06153b3d5ec1540fac1c8fbc97b5d2b58a8e I mistakenly/accidentally a apt-get update. As https://wiki.debian.org/Multiarch/HOWTO explains, apt-get update is needed after adding a new architecture. --- .../definitions/systemvmtemplate/install_systemvm_packages.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/appliance/definitions/systemvmtemplate/install_systemvm_packages.sh b/tools/appliance/definitions/systemvmtemplate/install_systemvm_packages.sh index dac727c4a8d..bc29e62c1b5 100644 --- a/tools/appliance/definitions/systemvmtemplate/install_systemvm_packages.sh +++ b/tools/appliance/definitions/systemvmtemplate/install_systemvm_packages.sh @@ -47,6 +47,7 @@ function install_packages() { #32 bit architecture support:: not required for 32 bit template if [ "${arch}" != "i386" ]; then dpkg --add-architecture i386 + apt-get update ${apt_install} links:i386 libuuid1:i386 fi From d658fc4637becee12267f670ee97b429e05fd655 Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Wed, 23 Jul 2014 17:34:26 +0200 Subject: [PATCH 48/60] CLOUDSTACK-7143: attempt to export a vagrant box --- tools/appliance/build.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/appliance/build.sh b/tools/appliance/build.sh index d315b43e20e..b0bea414294 100755 --- a/tools/appliance/build.sh +++ b/tools/appliance/build.sh @@ -283,7 +283,9 @@ function veewee_destroy() { function veewee_build() { log INFO "building new image with veewee" bundle exec veewee vbox build "${appliance_build_name}" ${VEEWEE_BUILD_ARGS} - bundle exec veewee vbox halt "${appliance_build_name}" ${VEEWEE_ARGS} + # vbox export wants to run vbox halt itself, so don't halt! + # bundle exec veewee vbox halt "${appliance_build_name}" ${VEEWEE_ARGS} + bundle exec veewee vbox export "${appliance_build_name}" ${VEEWEE_ARGS} } function check_appliance_shutdown() { From be8b2d7c21838038ccc2740bccb4afeb6b3caf16 Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Fri, 25 Jul 2014 12:20:38 +0200 Subject: [PATCH 49/60] CLOUDSTACK-7143: more robust box cleanup Having experimented with many edge cases of running multiple build.sh commands in parallel / against busy virtualbox setups, the only really reliable way to produce consistent images is to not do these commands in parallel and to not do them while the machine is doing many other things. If virtualbox or the machine that hosts it is very busy, and/or it has a lot of disks it knows/knew about, and/or its tuesday, behavior may be a bit different. Realizing this reality, this commit adds some scripts that try really hard to set virtualbox back to known/healthy state before building. --- tools/appliance/Gemfile | 1 + tools/appliance/build.sh | 72 ++++++++++++++++++++++++++++-- tools/appliance/vbox_disk_clean.rb | 33 ++++++++++++++ tools/appliance/vbox_vm_clean.rb | 51 +++++++++++++++++++++ 4 files changed, 153 insertions(+), 4 deletions(-) create mode 100755 tools/appliance/vbox_disk_clean.rb create mode 100755 tools/appliance/vbox_vm_clean.rb diff --git a/tools/appliance/Gemfile b/tools/appliance/Gemfile index dae53be6dd0..b1f23416bb9 100644 --- a/tools/appliance/Gemfile +++ b/tools/appliance/Gemfile @@ -18,3 +18,4 @@ source 'https://rubygems.org' gem 'veewee', :git => 'https://github.com/jedi4ever/veewee.git' gem 'em-winrm' +gem 'sys-proctable' diff --git a/tools/appliance/build.sh b/tools/appliance/build.sh index b0bea414294..736768af392 100755 --- a/tools/appliance/build.sh +++ b/tools/appliance/build.sh @@ -37,6 +37,10 @@ Usage: (or use command line arg, default i386, other option amd64) * Set \$ssh_key to provide root ssh public key to inject (or use command line arg, default set in the veewee definition its authorized_keys.sh) + * Set \$clean_vbox to try pretty hard to remove all our vms and disk from + virtualbox before and after running the rest of the build. This should + not be needed since we try hard to use VBoxManage nicely, but, various + error conditions / timing issues are quite hard to fully contain * Set \$DEBUG=1 to enable debug logging * Set \$TRACE=1 to enable trace logging * Set \$VEEWEE_ARGS to pass veewee custom arguments @@ -115,6 +119,9 @@ export VM_ARCH="${arch}" # server control ssh_key="${6:-${ssh_key:-}}" +# whether to attempt to clean up all our virtualbox vms/disks before/after run +clean_vbox="${clean_vbox:-}" + # while building with vbox, we need a quite unique appliance name in order to prevent conflicts with multiple # concurrent executors on jenkins if [ -z "${branch}" ] ; then @@ -266,6 +273,17 @@ function setup_ruby() { bundle check || bundle install ${bundle_args} } +function stop_vbox() { + log INFO "stoppping all virtualbox vms for ${USER}" + bundle exec ./vbox_vm_clean.rb +} + +function clean_vbox() { + log INFO "deleting all virtualbox vms and disks for ${USER}" + bundle exec ./vbox_vm_clean.rb --delete + bundle exec ./vbox_disk_clean.rb +} + function prepare() { log INFO "preparing for build" setup_ruby @@ -283,9 +301,11 @@ function veewee_destroy() { function veewee_build() { log INFO "building new image with veewee" bundle exec veewee vbox build "${appliance_build_name}" ${VEEWEE_BUILD_ARGS} - # vbox export wants to run vbox halt itself, so don't halt! - # bundle exec veewee vbox halt "${appliance_build_name}" ${VEEWEE_ARGS} - bundle exec veewee vbox export "${appliance_build_name}" ${VEEWEE_ARGS} +} + +function veewee_halt() { + log INFO "shutting down new vm with veewee" + bundle exec veewee vbox halt "${appliance_build_name}" ${VEEWEE_ARGS} } function check_appliance_shutdown() { @@ -300,6 +320,41 @@ function check_appliance_shutdown() { return ${result} } +function check_appliance_disk_ready() { + log INFO "waiting for veewee appliance disk to be available..." + # local hdd_path="vboxmanage showvminfo '${appliance_build_name}' --machinereadable | \ + # egrep '(SATA|IDE) Controller-[0-9]+-[0-9]+' | grep -v '.iso' | \ + # grep -v '="none"' | egrep -o '=".*"' | sed 's/=//' | sed 's/"//g'" + local hdd_path=`vboxmanage list hdds | grep "${appliance_build_name}\/" | grep vdi | \ + cut -c 14- | sed ${sed_regex_option} 's/^ *//'` + disk_state=`vboxmanage showhdinfo "${hdd_path}" | egrep '^State:' | sed 's/State://' | egrep -o '[a-zA-Z]+' | awk '{print tolower($0)}'` + if [ "${disk_state}" == "notcreated" ]; then + log ERROR "disk ${hdd_path} in state notcreated" + return 1 + elif [ "${disk_state}" == "created" ]; then + log INFO "disk ${hdd_path} in state created" + return 0 + elif [ "${disk_state}" == "lockedread" ]; then + log INFO "disk ${hdd_path} in state lockedread" + return 1 + elif [ "${disk_state}" == "lockedwrite" ]; then + log INFO "disk ${hdd_path} in state lockedwrite" + return 1 + elif [ "${disk_state}" == "inaccessible" ]; then + log INFO "disk ${hdd_path} in state inaccessible" + return 1 + elif [ "${disk_state}" == "creating" ]; then + log WARN "disk ${hdd_path} in state creating" + return 1 + elif [ "${disk_state}" == "deleting" ]; then + log WARN "disk ${hdd_path} in state deleting" + return 1 + else + log WARN "disk ${hdd_path} has unknown disk state ${disk_state}" + return 1 + fi +} + function remove_shares() { log INFO "removing shared folders from appliance..." set +e @@ -407,12 +462,21 @@ function hyperv_export() { ### function main() { + if [ "${clean_vbox}" == "1" ]; then + clean_vbox --delete + add_on_exit clean_vbox --delete + else + stop_vbox # some extra encouragement for virtualbox to stop things + fi prepare create_definition veewee_destroy # in case of left-over cruft from failed build add_on_exit veewee_destroy veewee_build + veewee_halt + stop_vbox # some extra encouragement for virtualbox to stop things retry 10 check_appliance_shutdown + retry 10 check_appliance_disk_ready retry 10 remove_shares # Get appliance uuids @@ -427,7 +491,7 @@ function main() { kvm_export "${hdd_path}" vmware_export "${machine_uuid}" "${hdd_uuid}" hyperv_export "${hdd_uuid}" - log INFO "BUILD SUCCESSFUL" + add_on_exit log INFO "BUILD SUCCESSFUL" } # we only run main() if not source-d diff --git a/tools/appliance/vbox_disk_clean.rb b/tools/appliance/vbox_disk_clean.rb new file mode 100755 index 00000000000..f7464e3b4dd --- /dev/null +++ b/tools/appliance/vbox_disk_clean.rb @@ -0,0 +1,33 @@ +#!/usr/bin/env ruby + +lines = `VBoxManage list hdds` +disks = lines.split(/\n\s*\n/) +disks.each do |disk| + disk_lines = disk.split(/\n/) + disk_config = {} + disk_lines.each do |line| + pair = line.split(/:\s*/) + disk_config[pair[0]] = pair[1] + # if pair[0] == 'Location' + # location = pair[1] + + # if location.include? '/Snapshots/' + # disk_config['is_snapshot'] = true + # end + # if location.include? '/VirtualBox VMs/' + # disk_config['vm_name'] = location.split('/VirtualBox VMs/')[1].split('/')[0] + # disk_config['disk_name'] = location.split('/')[-1] + # disk_config['is_virtualbox_vm'] = true + # else + # disk_config['is_virtualbox_vm'] = false + # disk_config['disk_name'] = location.split('/')[-1] + # end + # end + end + + if disk_config.include? 'Location' + cmd="VBoxManage closemedium disk '#{disk_config['Location']}' --delete" + puts cmd + `#{cmd}` + end +end diff --git a/tools/appliance/vbox_vm_clean.rb b/tools/appliance/vbox_vm_clean.rb new file mode 100755 index 00000000000..bbbc85a5c08 --- /dev/null +++ b/tools/appliance/vbox_vm_clean.rb @@ -0,0 +1,51 @@ +#!/usr/bin/env ruby + +# script that tries hard to forcibly shut down all vms + +# gem install sys-proctable +require 'sys/proctable' + +include Sys + +do_delete = (ARGV.include? 'delete' or ARGV.include? '--delete' or ARGV.include? '-d') + +lines = `VBoxManage list vms` +vms = lines.split(/\n/) +vms.each do |vmline| + vm_info = /\"(.*)\"[^{]*\{(.*)\}/.match(vmline) + vm_name = vm_info[1] + vm_uuid = vm_info[2] + + cmd="VBoxManage controlvm #{vm_name} poweroff" + puts cmd + `#{cmd}` + if do_delete + sleep(1) + cmd="VBoxManage unregistervm #{vm_name} --delete" + puts cmd + `#{cmd}` + end + + sleep(1) + # ps x | grep VBoxHeadless | grep systemvm64template-4.4.0 | egrep -o '^\s*[0-9]+' | xargs kill + ProcTable.ps { |p| + next unless p.cmdline.include? "VBoxHeadless" + next unless p.cmdline.include? vm_name + # VBoxManage should only list _our_ vms, but just to be safe... + next unless p.ruid == Process.uid + + puts "kill -SIGKILL #{p.pid}" + begin + Process.kill("KILL", p.pid) + rescue => exception + puts exception.backtrace + end + sleep(5) + puts "kill -SIGTERM #{p.pid}" + begin + Process.kill("TERM", p.pid) + rescue => exception + puts exception.backtrace + end + } +end From 00b39de0f88b8369243e745b73e7513edfb8844a Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Fri, 25 Jul 2014 14:49:45 +0200 Subject: [PATCH 50/60] CLOUDSTACK-7143: more reliable support for vagrant box export Veewee supports exporting vagrant boxes out of virtualbox, out of the box. However, it assumes that it can export a disk if the shutdown of the vm that is using that disk has succeeded. This assumption is not strictly always true (see previous commit). So, we replicate the bit of logic in veewee for making vagrant boxes. This has the added side benefit of creating an .ovf export only once, rather than once for vmware and then again for vagrant. --- tools/appliance/build.sh | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/tools/appliance/build.sh b/tools/appliance/build.sh index 736768af392..2cefd793c78 100755 --- a/tools/appliance/build.sh +++ b/tools/appliance/build.sh @@ -146,6 +146,9 @@ fi appliance_build_name=${appliance}${branch_tag}${version_tag} +# mac address of the vm we create +mac_address= + ### ### Generic helper functions ### @@ -308,6 +311,14 @@ function veewee_halt() { bundle exec veewee vbox halt "${appliance_build_name}" ${VEEWEE_ARGS} } +function save_mac_address() { + log INFO "saving new vm mac address" + mac_address=`vboxmanage showvminfo --details --machinereadable ${appliance_build_name} | grep macaddress1= | sed 's/macaddress1=//' | sed 's/"//g'` + if [ "${mac_address}" == "" ]; then + error "Could not find mac address for appliance ${appliance_build_name}" + fi +} + function check_appliance_shutdown() { log INFO "waiting for veewee appliance to shut down..." ! (vboxmanage list runningvms | grep "${appliance_build_name}") @@ -446,6 +457,32 @@ function vmware_export() { log INFO "${appliance} exported for VMWare: dist/${appliance_build_name}-vmware.ova" } +function vagrant_export() { + log INFO "creating vagrant export" + # this is based on veewee export logic, but, we don't want to use veewee export, + # since it makes optimistic assumptions about VM shutdown/halt leading to available + # disks and the like + disk="${appliance_build_name}-vmware.ovf" + mkdir -p "box/${appliance_build_name}" + cp "${disk}" "box/${appliance_build_name}/box.ovf" + cat >box/${appliance_build_name}/Vagrantfile < Date: Fri, 25 Jul 2014 15:05:24 +0200 Subject: [PATCH 51/60] CLOUDSTACK-7143: try to fix jenkins problems with last 2 commits * bundle install needs to run before running the vbox cleaning scripts, so move prepare step before clean step * feature branches have / in their name which is a bad character to put into filenames --- tools/appliance/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/appliance/build.sh b/tools/appliance/build.sh index 2cefd793c78..96caf78ea5e 100755 --- a/tools/appliance/build.sh +++ b/tools/appliance/build.sh @@ -499,13 +499,13 @@ function hyperv_export() { ### function main() { + prepare if [ "${clean_vbox}" == "1" ]; then clean_vbox --delete add_on_exit clean_vbox --delete else stop_vbox # some extra encouragement for virtualbox to stop things fi - prepare create_definition veewee_destroy # in case of left-over cruft from failed build add_on_exit veewee_destroy From 598c472b7372e908fcf3a9d19463d3c8b4b3b0bd Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Fri, 25 Jul 2014 15:15:01 +0200 Subject: [PATCH 52/60] CLOUDSTACK-7143: remove erroneous backticks The backticks in the Vagrantfile template were getting evaluated by bash. This caused some harmless but confusing error messages to appear on running the build. Easy fix is to remove them. --- tools/appliance/build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/appliance/build.sh b/tools/appliance/build.sh index 96caf78ea5e..2d611c44b53 100755 --- a/tools/appliance/build.sh +++ b/tools/appliance/build.sh @@ -467,9 +467,9 @@ function vagrant_export() { cp "${disk}" "box/${appliance_build_name}/box.ovf" cat >box/${appliance_build_name}/Vagrantfile < Date: Mon, 28 Jul 2014 17:15:32 +0200 Subject: [PATCH 53/60] CLOUDSTACK-7143: Fixed box file for virtualbox --- tools/appliance/build.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/appliance/build.sh b/tools/appliance/build.sh index 2d611c44b53..12f5f896fdf 100755 --- a/tools/appliance/build.sh +++ b/tools/appliance/build.sh @@ -463,8 +463,10 @@ function vagrant_export() { # since it makes optimistic assumptions about VM shutdown/halt leading to available # disks and the like disk="${appliance_build_name}-vmware.ovf" + image="${appliance_build_name}-vmware-disk1.vmdk" mkdir -p "box/${appliance_build_name}" cp "${disk}" "box/${appliance_build_name}/box.ovf" + cp "${image}" "box/${appliance_build_name}/box-disk1.vmdk" cat >box/${appliance_build_name}/Vagrantfile < Date: Tue, 29 Jul 2014 10:12:00 +0200 Subject: [PATCH 54/60] CLOUDSTACK-7143: work on vagrant box export Running --export creates the .ovf and the .vmdk files referenced from that .ovf in one go. Guessing/predicting the names of the .vmdk files is not fool-proof. --- tools/appliance/build.sh | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/tools/appliance/build.sh b/tools/appliance/build.sh index 12f5f896fdf..e3197e88db8 100755 --- a/tools/appliance/build.sh +++ b/tools/appliance/build.sh @@ -459,20 +459,24 @@ function vmware_export() { function vagrant_export() { log INFO "creating vagrant export" + local machine_uuid="${1}" # this is based on veewee export logic, but, we don't want to use veewee export, # since it makes optimistic assumptions about VM shutdown/halt leading to available # disks and the like - disk="${appliance_build_name}-vmware.ovf" - image="${appliance_build_name}-vmware-disk1.vmdk" + # predicting VBoxManage disk naming seems problematic + # disk="${appliance_build_name}-vmware.ovf" + # image="${appliance_build_name}-vmware-disk1.vmdk" mkdir -p "box/${appliance_build_name}" - cp "${disk}" "box/${appliance_build_name}/box.ovf" - cp "${image}" "box/${appliance_build_name}/box-disk1.vmdk" + # cp "${disk}" "box/${appliance_build_name}/box.ovf" + # cp "${image}" "box/${appliance_build_name}/box-disk1.vmdk" cat >box/${appliance_build_name}/Vagrantfile < Date: Thu, 31 Jul 2014 15:23:18 +0200 Subject: [PATCH 55/60] CLOUDSTACK-7143: only archive the zipped hyperv image Cherry-pick of d09acc44f2b36cd593ebe97ab1673b7757ea2c25. --- tools/appliance/build.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/appliance/build.sh b/tools/appliance/build.sh index e3197e88db8..db21dade21d 100755 --- a/tools/appliance/build.sh +++ b/tools/appliance/build.sh @@ -497,7 +497,8 @@ function hyperv_export() { # HyperV doesn't support import a zipped image from S3, # but we create a zipped version to save space on the jenkins box zip "${appliance_build_name}-hyperv.vhd.zip" "${appliance_build_name}-hyperv.vhd" - mv "${appliance_build_name}-hyperv.vhd.zip" "${appliance_build_name}-hyperv.vhd" dist/ + add_on_exit rm "${appliance_build_name}-hyperv.vhd" + mv "${appliance_build_name}-hyperv.vhd.zip" dist/ log INFO "${appliance} exported for HyperV: dist/${appliance_build_name}-hyperv.vhd.zip" } From a8500ed73ea6873812e10e258d5a7db8446b33eb Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Thu, 4 Sep 2014 15:22:28 +0200 Subject: [PATCH 56/60] CLOUDSTACK-7143: survive / in branch names --- tools/appliance/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/appliance/build.sh b/tools/appliance/build.sh index db21dade21d..f9c6b75d306 100755 --- a/tools/appliance/build.sh +++ b/tools/appliance/build.sh @@ -125,7 +125,7 @@ clean_vbox="${clean_vbox:-}" # while building with vbox, we need a quite unique appliance name in order to prevent conflicts with multiple # concurrent executors on jenkins if [ -z "${branch}" ] ; then - branch=`(git name-rev --no-undefined --name-only HEAD 2>/dev/null || echo unknown) | sed -e 's/remotes\/.*\///g'` + branch=`(git name-rev --no-undefined --name-only HEAD 2>/dev/null || echo unknown) | sed -e 's/remotes\/.*\///g' | sed -e 's|/|_|g'` fi branch_tag= From 362946c52e21b70b42b716c8d60431432223f22d Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Wed, 17 Sep 2014 12:24:00 +0200 Subject: [PATCH 57/60] CLOUDSTACK-7143: don't call rvm from build It is simpler to expect that rvm setup is done outside of this build. The buildacloud.org jenkins has rvm installed/enabled by default so does not invoke rvm. --- tools/appliance/README.md | 26 +++++++++++++++++++++++++- tools/appliance/build.sh | 15 ++++----------- 2 files changed, 29 insertions(+), 12 deletions(-) diff --git a/tools/appliance/README.md b/tools/appliance/README.md index 8c51c9ca756..411fd09a76f 100644 --- a/tools/appliance/README.md +++ b/tools/appliance/README.md @@ -40,12 +40,36 @@ Note, gem may require gcc-4.2, make sure link exists: sudo ln -s /usr/bin/gcc /usr/bin/gcc-4.2 +# Setting up jenkins (CI) builds + +All the tools listed above are expected to be available. If you follow + + http://rvm.io/integration/jenkins + +then you'll need to do a bit of logic to load RVM in jenkins. In the +build script you put into jenkins, start it with +``` +#!/bin/bash -l +``` + +to ensure a login shell, then add something like +``` +# inspired by https://github.com/CloudBees-community/rubyci-clickstart/blob/master/bin/run-ci +# also see https://rvm.io/integration/jenkins +# .rvmrc won't get trusted/auto-loaded by jenkins by default +export VAGRANT_HOME=$HOME/.vagrant.d-release-cloudstack +rvm use ruby-1.9.3@vagrant-release-cloudstack --create +# do not use --deployment since that requires Gemfile.lock...and we prefer an up-to-date veewee +bundle_args="--path vendor/bundle" +``` + + # How to build SystemVMs automatically Just run build.sh, it will export archived appliances for KVM, XenServer, VMWare and HyperV in `dist`: - sh build.sh [systemvmtemplate|systemvmtemplate64] + bash build.sh [systemvmtemplate|systemvmtemplate64] # Building SystemVM template appliance manually diff --git a/tools/appliance/build.sh b/tools/appliance/build.sh index f9c6b75d306..7880a1f58d2 100755 --- a/tools/appliance/build.sh +++ b/tools/appliance/build.sh @@ -97,6 +97,9 @@ if [[ "${VEEWEE_ARGS}" == "" && "${TRACE}" == "1" ]]; then fi VEEWEE_BUILD_ARGS="${VEEWEE_BUILD_ARGS:-${VEEWEE_ARGS} --nogui --auto}" +# any arguments to pass along to the 'bundle install' command +BUNDLE_ARGS="${BUNDLE_ARGS:-}" + # which veewee definition to use appliance="${1:-${appliance:-systemvmtemplate}}" @@ -263,17 +266,7 @@ function create_definition() { } function setup_ruby() { - local bundle_args= - if [[ ! -z "${JENKINS_HOME}" ]]; then - # inspired by https://github.com/CloudBees-community/rubyci-clickstart/blob/master/bin/run-ci - # also see https://rvm.io/integration/jenkins - # .rvmrc won't get trusted/auto-loaded by jenkins by default - export VAGRANT_HOME=$HOME/.vagrant.d-release-cloudstack - rvm use ruby-1.9.3@vagrant-release-cloudstack --create - # do not use --deployment since that requires Gemfile.lock...and we prefer an up-to-date veewee - bundle_args="--path vendor/bundle" - fi - bundle check || bundle install ${bundle_args} + bundle check || bundle install ${BUNDLE_ARGS} } function stop_vbox() { From 64f4fb1af8efafd8f5548cc19c09d3c8fab3402e Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Thu, 18 Sep 2014 11:41:47 +0200 Subject: [PATCH 58/60] CLOUDSTACK-7143: note shar dependency --- tools/appliance/README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/appliance/README.md b/tools/appliance/README.md index 411fd09a76f..7a9973ae70d 100644 --- a/tools/appliance/README.md +++ b/tools/appliance/README.md @@ -19,9 +19,11 @@ under the License. # Setting up Tools and Environment - - Install VirtualBox 4.2 or latest - - Tool for exporting appliances: qemu-img, vboxmanage, vhd-util + - Install latest VirtualBox (at least 4.2) + - Install tools for exporting appliances: qemu-img, vboxmanage, vhd-util - Install [RVM](https://rvm.io/rvm/install) + - Install shar + yum install sharutils - Setup paths: export PATH=~/.rvm/bin:$PATH - Install Ruby 1.9.3, if it installed some other version: From e43e08389328aa45689f184ec34509437c11c3be Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Fri, 19 Sep 2014 15:15:16 +0200 Subject: [PATCH 59/60] CLOUDSTACK-7143: protect against old ruby --- tools/appliance/vbox_vm_clean.rb | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tools/appliance/vbox_vm_clean.rb b/tools/appliance/vbox_vm_clean.rb index bbbc85a5c08..a0e34e619ca 100755 --- a/tools/appliance/vbox_vm_clean.rb +++ b/tools/appliance/vbox_vm_clean.rb @@ -31,8 +31,11 @@ vms.each do |vmline| ProcTable.ps { |p| next unless p.cmdline.include? "VBoxHeadless" next unless p.cmdline.include? vm_name - # VBoxManage should only list _our_ vms, but just to be safe... - next unless p.ruid == Process.uid + # not all rubies / proctables expose ruid + if defined? p.ruid + # VBoxManage should only list _our_ vms, but just to be safe... + next unless p.ruid == Process.uid + end puts "kill -SIGKILL #{p.pid}" begin From ba009ed51a12a8ff3a31b150d488bd632bc10803 Mon Sep 17 00:00:00 2001 From: Leo Simons Date: Mon, 22 Sep 2014 18:10:56 +0200 Subject: [PATCH 60/60] CLOUDSTACK-7143: attempt to be safer when cleaning up --- tools/appliance/build.sh | 9 +++--- tools/appliance/vbox_vm_clean.rb | 49 +++++++++++++++++--------------- 2 files changed, 30 insertions(+), 28 deletions(-) diff --git a/tools/appliance/build.sh b/tools/appliance/build.sh index 7880a1f58d2..fd24a436103 100755 --- a/tools/appliance/build.sh +++ b/tools/appliance/build.sh @@ -276,7 +276,7 @@ function stop_vbox() { function clean_vbox() { log INFO "deleting all virtualbox vms and disks for ${USER}" - bundle exec ./vbox_vm_clean.rb --delete + bundle exec ./vbox_vm_clean.rb --delete --kill bundle exec ./vbox_disk_clean.rb } @@ -502,10 +502,10 @@ function hyperv_export() { function main() { prepare if [ "${clean_vbox}" == "1" ]; then - clean_vbox --delete - add_on_exit clean_vbox --delete + clean_vbox + add_on_exit clean_vbox else - stop_vbox # some extra encouragement for virtualbox to stop things + stop_vbox fi create_definition veewee_destroy # in case of left-over cruft from failed build @@ -513,7 +513,6 @@ function main() { veewee_build save_mac_address veewee_halt - stop_vbox # some extra encouragement for virtualbox to stop things retry 10 check_appliance_shutdown retry 10 check_appliance_disk_ready retry 10 remove_shares diff --git a/tools/appliance/vbox_vm_clean.rb b/tools/appliance/vbox_vm_clean.rb index a0e34e619ca..0c0c27af98b 100755 --- a/tools/appliance/vbox_vm_clean.rb +++ b/tools/appliance/vbox_vm_clean.rb @@ -8,6 +8,7 @@ require 'sys/proctable' include Sys do_delete = (ARGV.include? 'delete' or ARGV.include? '--delete' or ARGV.include? '-d') +do_kill = (ARGV.include? 'kill' or ARGV.include? '--kill' or ARGV.include? '-k') lines = `VBoxManage list vms` vms = lines.split(/\n/) @@ -26,29 +27,31 @@ vms.each do |vmline| `#{cmd}` end - sleep(1) - # ps x | grep VBoxHeadless | grep systemvm64template-4.4.0 | egrep -o '^\s*[0-9]+' | xargs kill - ProcTable.ps { |p| - next unless p.cmdline.include? "VBoxHeadless" - next unless p.cmdline.include? vm_name - # not all rubies / proctables expose ruid - if defined? p.ruid - # VBoxManage should only list _our_ vms, but just to be safe... - next unless p.ruid == Process.uid - end + if do_kill + sleep(1) + # ps x | grep VBoxHeadless | grep systemvm64template-4.4.0 | egrep -o '^\s*[0-9]+' | xargs kill + ProcTable.ps do |p| + next unless p.cmdline.include? "VBoxHeadless" + next unless p.cmdline.include? vm_name + # not all rubies / proctables expose ruid + if defined? p.ruid + # VBoxManage should only list _our_ vms, but just to be safe... + next unless p.ruid == Process.uid + end - puts "kill -SIGKILL #{p.pid}" - begin - Process.kill("KILL", p.pid) - rescue => exception - puts exception.backtrace + puts "kill -SIGKILL #{p.pid}" + begin + Process.kill("KILL", p.pid) + rescue => exception + puts exception.backtrace + end + sleep(5) + puts "kill -SIGTERM #{p.pid}" + begin + Process.kill("TERM", p.pid) + rescue => exception + puts exception.backtrace + end end - sleep(5) - puts "kill -SIGTERM #{p.pid}" - begin - Process.kill("TERM", p.pid) - rescue => exception - puts exception.backtrace - end - } + end end