From ba2fc978654be46008dd3aaa7e33fcc4564eb227 Mon Sep 17 00:00:00 2001
From: Sheng Yang <sheng.yang@cloud.com>
Date: Wed, 14 Sep 2011 02:58:45 -0700
Subject: [PATCH] bug 11351: Add monitor process for keepalived

Then when the process dead, we can know it and prevent two MASTER case happened.
---
 .../config/etc/init.d/cloud-early-config      | 10 ++++++++++
 .../redundant_router/check_heartbeat.sh.templ | 20 +++++++++++++++++++
 .../root/redundant_router/heartbeat.sh.templ  |  4 ++++
 .../redundant_router/keepalived.conf.templ    |  6 ++++++
 4 files changed, 40 insertions(+)
 create mode 100755 patches/systemvm/debian/config/root/redundant_router/check_heartbeat.sh.templ
 create mode 100755 patches/systemvm/debian/config/root/redundant_router/heartbeat.sh.templ

diff --git a/patches/systemvm/debian/config/etc/init.d/cloud-early-config b/patches/systemvm/debian/config/etc/init.d/cloud-early-config
index 6c18e0f7850..0e32ff5c99f 100755
--- a/patches/systemvm/debian/config/etc/init.d/cloud-early-config
+++ b/patches/systemvm/debian/config/etc/init.d/cloud-early-config
@@ -348,6 +348,8 @@ setup_redundant_router() {
     cp /root/redundant_router/backup.sh.templ $rrouter_bin_path/backup.sh
     cp /root/redundant_router/fault.sh.templ $rrouter_bin_path/fault.sh
     cp /root/redundant_router/primary-backup.sh.templ $rrouter_bin_path/primary-backup.sh
+    cp /root/redundant_router/heartbeat.sh.templ $rrouter_bin_path/heartbeat.sh
+    cp /root/redundant_router/check_heartbeat.sh.templ $rrouter_bin_path/check_heartbeat.sh
     cp /root/redundant_router/checkrouter.sh.templ /root/checkrouter.sh
     sed -i "s/\[ROUTER_ID\]/$NAME/g" /etc/keepalived/keepalived.conf
     sed -i "s/\[ROUTER_IP\]/$GUEST_GW\/$GUEST_CIDR_SIZE/g" /etc/keepalived/keepalived.conf
@@ -366,10 +368,13 @@ setup_redundant_router() {
     sed -i "s/\[RROUTER_BIN_PATH\]/$rrouter_bin_path_str/g" $rrouter_bin_path/master.sh
     sed -i "s/\[RROUTER_BIN_PATH\]/$rrouter_bin_path_str/g" $rrouter_bin_path/backup.sh
     sed -i "s/\[RROUTER_BIN_PATH\]/$rrouter_bin_path_str/g" $rrouter_bin_path/fault.sh
+    sed -i "s/\[RROUTER_BIN_PATH\]/$rrouter_bin_path_str/g" $rrouter_bin_path/heartbeat.sh
+    sed -i "s/\[RROUTER_BIN_PATH\]/$rrouter_bin_path_str/g" $rrouter_bin_path/check_heartbeat.sh
     sed -i "s/\[RROUTER_LOG\]/$rrouter_log_str/g" $rrouter_bin_path/master.sh
     sed -i "s/\[RROUTER_LOG\]/$rrouter_log_str/g" $rrouter_bin_path/backup.sh
     sed -i "s/\[RROUTER_LOG\]/$rrouter_log_str/g" $rrouter_bin_path/fault.sh
     sed -i "s/\[RROUTER_LOG\]/$rrouter_log_str/g" $rrouter_bin_path/primary-backup.sh
+    sed -i "s/\[RROUTER_LOG\]/$rrouter_log_str/g" $rrouter_bin_path/check_heartbeat.sh
     sed -i "s/\[RROUTER_LOG\]/$rrouter_log_str/g" /root/checkrouter.sh
     chmod a+x $rrouter_bin_path/*.sh
 
@@ -379,6 +384,11 @@ setup_redundant_router() {
     then
         sed -i "s/if\ start-stop-daemon\ --start/sleep\ 10;if\ start-stop-daemon\ --start/g" /etc/init.d/keepalived
     fi
+    crontab -l|grep "check_heartbeat.sh"
+    if [ $? -ne 0 ]
+    then
+        (crontab -l; echo "*/1 * * * * $rrouter_bin_path/check_heartbeat.sh 2>&1 > /dev/null") | crontab
+    fi
 }
 
 setup_router() {
diff --git a/patches/systemvm/debian/config/root/redundant_router/check_heartbeat.sh.templ b/patches/systemvm/debian/config/root/redundant_router/check_heartbeat.sh.templ
new file mode 100755
index 00000000000..221c669e55a
--- /dev/null
+++ b/patches/systemvm/debian/config/root/redundant_router/check_heartbeat.sh.templ
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+if [ -e [RROUTER_BIN_PATH]/keepalived.ts2 ]
+then
+    lasttime=$(cat [RROUTER_BIN_PATH]/keepalived.ts2)
+    thistime=$(cat [RROUTER_BIN_PATH]/keepalived.ts)
+    diff=$(($thistime - $lasttime))
+    if [ $diff -gt 100 ]
+    then
+        echo Keepalived process is dead! >> [RROUTER_LOG]
+        service keepalived stop >> [RROUTER_LOG] 2>&1
+        service conntrackd stop >> [RROUTER_LOG] 2>&1
+	pkill -9 keepalived >> [RROUTER_LOG] 2>&1
+        ifconfig eth2 down >> [RROUTER_LOG] 2>&1
+        echo Status: FAULT \(keepalived process is dead\) >> [RROUTER_LOG]
+        exit
+    fi
+fi
+
+cp [RROUTER_BIN_PATH]/keepalived.ts [RROUTER_BIN_PATH]/keepalived.ts2
diff --git a/patches/systemvm/debian/config/root/redundant_router/heartbeat.sh.templ b/patches/systemvm/debian/config/root/redundant_router/heartbeat.sh.templ
new file mode 100755
index 00000000000..90d790b5372
--- /dev/null
+++ b/patches/systemvm/debian/config/root/redundant_router/heartbeat.sh.templ
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+t=$(date +%s)
+echo $t > [RROUTER_BIN_PATH]/keepalived.ts
diff --git a/patches/systemvm/debian/config/root/redundant_router/keepalived.conf.templ b/patches/systemvm/debian/config/root/redundant_router/keepalived.conf.templ
index 2f400badf9f..fd7235f67e2 100644
--- a/patches/systemvm/debian/config/root/redundant_router/keepalived.conf.templ
+++ b/patches/systemvm/debian/config/root/redundant_router/keepalived.conf.templ
@@ -10,6 +10,11 @@ vrrp_script check_bumpup {
     weight [DELTA]
 }
 
+vrrp_script heartbeat {
+    script "[RROUTER_BIN_PATH]/heartbeat.sh"
+    interval 10
+}
+
 vrrp_instance inside_network {
     state BACKUP
     interface eth0
@@ -28,6 +33,7 @@ vrrp_instance inside_network {
 
     track_script {
         check_bumpup
+        heartbeat
     }
 
     #nopreempt