From dc535b96a2eeeaeffa1a2833dd4d39527541e3d5 Mon Sep 17 00:00:00 2001
From: alena <alena@cloud.com>
Date: Tue, 28 Jun 2011 11:02:14 -0700
Subject: [PATCH] bug 10475: cloud-sysvmadm - stop routers in multiple threads.
 Default number of threads is 5. status 10475: resolved fixed

Also added support for logging. By default the logs go to cloud.log file under current dir; you can specify another log location using -l option
---
 setup/bindir/cloud-sysvmadm.in | 153 ++++++++++++++++++++++++---------
 1 file changed, 111 insertions(+), 42 deletions(-)

diff --git a/setup/bindir/cloud-sysvmadm.in b/setup/bindir/cloud-sysvmadm.in
index 33db48bcf25..70f186deac4 100755
--- a/setup/bindir/cloud-sysvmadm.in
+++ b/setup/bindir/cloud-sysvmadm.in
@@ -5,7 +5,7 @@
 #set -x
 
 usage() {
-  printf "\nThe tool stopping/starting running system vms and domain routers \n\nUsage: %s: [-d] [-u] [-p] [-m] [-s] [-r] [-a] \n\n -d - cloud DB server ip address, defaulted to localhost if not specified \n -u - user name to access cloud DB, defaulted to "root" if not specified \n -p - cloud DB user password, defaulted to no password if not specified \n\n -m - the ip address of management server, defaulted to localhost if not specified\n\n -s - stop then start all running SSVMs and Console Proxies \n -r - stop then start all running Virtual Routers\n -a - stop then start all running SSVMs, Console Proxies, and Virtual Routers\n\n" $(basename $0) >&2
+  printf "\nThe tool stopping/starting running system vms and domain routers \n\nUsage: %s: [-d] [-u] [-p] [-m] [-s] [-r] [-a] [-t]\n\n -d - cloud DB server ip address, defaulted to localhost if not specified \n -u - user name to access cloud DB, defaulted to "root" if not specified \n -p - cloud DB user password, defaulted to no password if not specified \n\n -m - the ip address of management server, defaulted to localhost if not specified\n\n -s - stop then start all running SSVMs and Console Proxies \n -r - stop then start all running Virtual Routers\n -a - stop then start all running SSVMs, Console Proxies, and Virtual Routers \n -t - number of parallel threads used for stopping Domain Routers. Default is 5.\n -l - log file location. Default is cloud.log under current directory.\n\n" $(basename $0) >&2
 }
 
 
@@ -17,9 +17,11 @@ ms=localhost
 user=root
 password=
 help=
+maxthreads=5
+LOGFILE=cloud.log
 
 
-while getopts 'sarhd:m:u:p:' OPTION
+while getopts 'sarhd:m:u:p:t:l:' OPTION
 do
   case $OPTION in
   s)    system=1
@@ -37,9 +39,15 @@ do
   h)    help=1
         ;;
   m)    ms="$OPTARG"
+  		;;
+  t)	maxthreads="$OPTARG"
+  		;;
+  l)	LOGFILE="$OPTARG"
   esac
 done
 
+ 
+
 
 stop_start_system() {
 secondary=(`mysql -h $db --user=$user --password=$password --skip-column-names -U cloud -e "select id from vm_instance where state=\"Running\" and type=\"SecondaryStorageVm\""`)
@@ -49,14 +57,16 @@ length_console=(${#console[@]})
 
 
 echo -e "\nStopping and starting secondary storage vms..."
+echo -e "Stopping and starting secondary storage vms..." >>$LOGFILE
+
 for d in "${secondary[@]}"; do
 	jobresult=$(send_request stopSystemVm $d)
 	if [ "$jobresult" != "1" ]; then
-		echo "ERROR: Failed to stop secondary storage vm with id $d"
+		echo "ERROR: Failed to stop secondary storage vm with id $d" >>$LOGFILE
 	else	
-		jobresult=$(send_request startSystemVm $d)
+		jobresult=$(send_request startSystemVm $d SSVM)
 		if [ "$jobresult" != "1" ]; then
-                	echo "ERROR: Failed to start secondary storage vm with id $d"
+        	echo "ERROR: Failed to start secondary storage vm with id $d" >>$LOGFILE
 		fi
 	fi
 done
@@ -64,52 +74,91 @@ done
 if [ "$length_secondary" == "0" ];then
 	echo -e "No running secondary storage vms found \n"
 else 
-	echo -e "Done \n"
+	echo -e "Done stopping and starting secondary storage vms"
+	echo -e "Done stopping and starting secondary storage vms." >>$LOGFILE
 fi
 
-echo "Stopping and starting console proxy vms..."
+echo -e "\nStopping and starting console proxy vms..."
+echo -e "Stopping and starting console proxy vms..." >>$LOGFILE
+
 for d in "${console[@]}"; do
     jobresult=$(send_request stopSystemVm $d)
 	if [ "$jobresult" != "1" ]; then
-                echo "ERROR: Failed to stop console proxy vm with id $d"
-        else	
-		jobresult=$(send_request startSystemVm $d)
-                if [ "$jobresult" != "1" ]; then
-                        echo "ERROR: Failed to start console proxy vm with id $d"
-                fi
-        fi
+            echo "ERROR: Failed to stop console proxy vm with id $d" >>$LOGFILE
+    else	
+			jobresult=$(send_request startSystemVm $d consoleProxy)
+    fi
 done
 
 if [ "$length_console" == "0" ];then
         echo -e "No running console proxy vms found \n"
 else
-        echo -e "Done \n"
+        echo "Done stopping and starting console proxy vms."
+		echo "Done stopping and starting console proxy vms." >>$LOGFILE
 fi
 }
 
-
 stop_start_router() {
-router=(`mysql -h $db --user=$user --password=$password --skip-column-names -U cloud -e "select id from vm_instance where state=\"Running\" and type=\"DomainRouter\""`)
-length_router=(${#router[@]})
+	router=(`mysql -h $db --user=$user --password=$password --skip-column-names -U cloud -e "select id from vm_instance where state=\"Running\" and type=\"DomainRouter\""`)
+	length_router=(${#router[@]})
+	
+	echo -e "\nStopping and starting running routing vms... "
+	echo -e "Stopping and starting running routing vms... " >>$LOGFILE
+	
+	#Spawn reboot router in parallel - run commands in <n> chunks - number of threads is configurable
 
-echo -e "\nStopping and starting running routing vms..."
-for d in "${router[@]}"; do
-        jobresult=$(send_request stopRouter $d)
-        if [ "$jobresult" != "1" ]; then
-                echo "ERROR: Failed to stop domain router with id $d"
-        else
-		jobresult=$(send_request startRouter $d)
-                if [ "$jobresult" != "1" ]; then
-                        echo "ERROR: Failed to start domain router with id $d"
-                fi
-        fi
-done
+	pids=()
+	for d in "${router[@]}"; do
+        
+    	reboot_router $d &
 
-if [ "$length_router" == "0" ];then
-        echo -e "No running router vms found \n"
-else
-        echo -e "Done \n"
-fi
+    	pids=( "${pids[@]}" $! )
+    	
+    	length_pids=(${#pids[@]})
+    	unfinishedPids=(${#pids[@]})
+    	
+    	if [ $maxthreads -gt $length_router ]; then
+    		maxthreads=$length_router
+    	fi
+  		
+  		if [ $length_pids -ge $maxthreads ]; then
+  			while [ $unfinishedPids -gt 0 ]; do
+    			sleep 10
+    			count=0
+	    		for ((  i = 0 ;  i < $length_pids;  i++  )); do
+	    			if ! ps ax | grep -v grep | grep ${pids[$i]} > /dev/null; then
+	    				count=`expr $count + 1`
+	    			fi
+	    		done
+	    		
+	    		if [ $count -eq $unfinishedPids ]; then
+	    			unfinishedPids=0
+	    		fi
+	    		
+    		done
+    		
+    		#remove all elements from pids
+	    	if [ $unfinishedPids -eq 0 ]; then
+	    		pids=()
+	    		length_pids=(${#pids[@]})
+	    	fi
+	    	
+  		fi
+  		
+	done
+
+	
+	if [ "$length_router" == "0" ];then
+		echo -e "No running router vms found \n" >>$LOGFILE
+	else	
+		while [ $unfinishedPids -gt 0 ]; do
+			sleep 10
+		done
+		
+		echo -e "Done restarting routers. \n"
+		echo -e "Done restarting routers. \n" >>$LOGFILE
+		
+	fi
 }
 
 stop_start_all() {
@@ -118,15 +167,35 @@ stop_start_router
 }
 
 send_request(){
-jobid=`curl -sS "http://$ms:8096/?command=$1&id=$2&response=json" | sed 's/\"//g' | sed 's/ //g' | sed 's/{//g' | sed 's/}//g' | awk -F: {'print $3'}`
-if [ "$jobid" == "" ]; then
-	echo 2
-	return
-fi
-jobresult=$(query_async_job_result $jobid)
-echo $jobresult
+	jobid=`curl -sS "http://$ms:8096/?command=$1&id=$2&response=json" | sed 's/\"//g' | sed 's/ //g' | sed 's/{//g' | sed 's/}//g' | awk -F: {'print $3'}`
+	if [ "$jobid" == "" ]; then
+		echo 2
+		return
+	fi
+	jobresult=$(query_async_job_result $jobid)
+	echo $jobresult
 }
 
+
+reboot_router(){
+	jobid=`curl -sS "http://$ms:8096/?command=rebootRouter&id=$1&response=json" | sed 's/\"//g' | sed 's/ //g' | sed 's/{//g' | sed 's/}//g' | awk -F: {'print $3'}`
+	if [ "$jobid" == "" ]; then
+		echo "ERROR: Failed to restart domainRouter with id $1" >>$LOGFILE
+		echo 2
+		return
+	fi
+	
+	jobresult=$(query_async_job_result $jobid)
+	
+	if [ "$jobresult" != "1" ]; then
+		echo "ERROR: Failed to restart domainRouter with id $1" >>$LOGFILE
+	else
+		echo "INFO: Successfully restarted domainRouter with id $1" >>$LOGFILE
+	fi
+	
+}
+
+
 query_async_job_result() {
 while [ 1 ]
 do