From 925f5f2f0959bf675f40864d27d4db1fc06f8ccf Mon Sep 17 00:00:00 2001
From: Chiradeep Vittal <chiradeep@cloud.com>
Date: Tue, 30 Aug 2011 21:49:22 -0700
Subject: [PATCH] Retry 3 times on deadlock: if not updated, then throw a
 runtime exception to roll back the vm start

---
 .../security/SecurityGroupManagerImpl2.java      |  4 ++++
 .../security/dao/VmRulesetLogDaoImpl.java        | 16 ++++++++++++----
 2 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/server/src/com/cloud/network/security/SecurityGroupManagerImpl2.java b/server/src/com/cloud/network/security/SecurityGroupManagerImpl2.java
index c02783fe0a6..cab68b25cc5 100644
--- a/server/src/com/cloud/network/security/SecurityGroupManagerImpl2.java
+++ b/server/src/com/cloud/network/security/SecurityGroupManagerImpl2.java
@@ -31,6 +31,7 @@ import com.cloud.exception.AgentUnavailableException;
 import com.cloud.network.security.SecurityGroupWork.Step;
 import com.cloud.uservm.UserVm;
 import com.cloud.utils.Profiler;
+import com.cloud.utils.exception.CloudRuntimeException;
 import com.cloud.vm.VirtualMachine.State;
 
 /**
@@ -98,6 +99,9 @@ public class SecurityGroupManagerImpl2 extends SecurityGroupManagerImpl {
         int updated = 0;
         if (updateSeqno) {
             updated = _rulesetLogDao.createOrUpdate(workItems);
+            if (updated < workItems.size()) {
+                throw new CloudRuntimeException("Failed to create ruleset log entries");
+            }
         }
         int newJobs = _workQueue.submitWorkForVms(workItems);
         p.stop();
diff --git a/server/src/com/cloud/network/security/dao/VmRulesetLogDaoImpl.java b/server/src/com/cloud/network/security/dao/VmRulesetLogDaoImpl.java
index 791145c4ff5..22e0cf27b91 100644
--- a/server/src/com/cloud/network/security/dao/VmRulesetLogDaoImpl.java
+++ b/server/src/com/cloud/network/security/dao/VmRulesetLogDaoImpl.java
@@ -88,7 +88,7 @@ public class VmRulesetLogDaoImpl extends GenericDaoBase<VmRulesetLogVO, Long> im
     private int executeWithRetryOnDeadlock(Transaction txn, String pstmt,  List<Long> vmIds) throws SQLException {
 
         int numUpdated = 0;
-        final int maxTries = 2;
+        final int maxTries = 3;
         for (int i=0; i < maxTries; i++) {
             try {
                 PreparedStatement stmtInsert = txn.prepareAutoCloseStatement(pstmt);
@@ -99,8 +99,15 @@ public class VmRulesetLogDaoImpl extends GenericDaoBase<VmRulesetLogVO, Long> im
                 numUpdated = stmtInsert.executeUpdate();
                 i = maxTries;
             } catch (SQLTransactionRollbackException e1) {
-                if (i < maxTries-1)
-                    s_logger.debug("Caught a deadlock exception while inserting security group rule log, retrying");
+                if (i < maxTries-1) {
+                    int delayMs =  (i+1)*1000;
+                    s_logger.debug("Caught a deadlock exception while inserting security group rule log, retrying in " + delayMs);
+                    try {
+                        Thread.sleep(delayMs);
+                    } catch(InterruptedException ie) {
+                        
+                    }
+                }
                 else 
                     s_logger.warn("Caught another deadlock exception while retrying inserting security group rule log, giving up");
 
@@ -134,7 +141,8 @@ public class VmRulesetLogDaoImpl extends GenericDaoBase<VmRulesetLogVO, Long> im
                         if (s_logger.isTraceEnabled()) {
                             s_logger.trace("Inserted or updated " + numUpdated + " rows");
                         }
-                        count += stmtSize;
+                        if (numUpdated > 0)
+                            count += stmtSize;
                     }
                     remaining = remaining - numStmts * stmtSize;
                 }