CLOUDSTACK-9571: Fence DB if there are consecutive connection errors.

client/tomcatconf/db.properties.in

@@ -27,6 +27,8 @@ db.cloud.password=@DBPW@
 db.cloud.host=@DBHOST@
 db.cloud.port=3306
 db.cloud.name=cloud
+db.connection.errors.fence=true
+db.connection.error.tolerance=2
 
 # CloudStack database tuning parameters
 db.cloud.maxActive=250

framework/db/resources/META-INF/cloudstack/system/spring-framework-db-system-context.xml

@@ -17,16 +17,15 @@
   under the License.
 -->
 <beans xmlns="http://www.springframework.org/schema/beans"
-       xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-       xmlns:context="http://www.springframework.org/schema/context"
-       xmlns:aop="http://www.springframework.org/schema/aop"
-       xsi:schemaLocation="http://www.springframework.org/schema/beans
+	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xmlns:context="http://www.springframework.org/schema/context"
+	xmlns:aop="http://www.springframework.org/schema/aop"
+	xsi:schemaLocation="http://www.springframework.org/schema/beans
                       http://www.springframework.org/schema/beans/spring-beans-3.0.xsd
                       http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop-3.0.xsd
                       http://www.springframework.org/schema/context
-                      http://www.springframework.org/schema/context/spring-context-3.0.xsd"
-                      >
+                      http://www.springframework.org/schema/context/spring-context-3.0.xsd">
+
+	<bean id="entityManagerImpl" class="com.cloud.dao.EntityManagerImpl" />
 
-    <bean id="entityManagerImpl" class="com.cloud.dao.EntityManagerImpl" />
-    
 </beans>

framework/db/src/com/cloud/utils/db/DbConnectionObserver.java

@@ -0,0 +1,8 @@
+package com.cloud.utils.db;
+
+import java.sql.SQLException;
+
+public interface DbConnectionObserver {
+    void onError(SQLException se);
+    void onSuccess();
+}

framework/db/src/com/cloud/utils/db/DbConnectionObserverImpl.java

@@ -0,0 +1,68 @@
+package com.cloud.utils.db;
+
+import java.sql.SQLException;
+import java.util.Properties;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.commons.lang.math.NumberUtils;
+import org.apache.log4j.Logger;
+
+import com.google.common.collect.ImmutableMap;
+
+public class DbConnectionObserverImpl implements DbConnectionObserver {
+    private static final Logger s_logger = Logger.getLogger(DbConnectionObserverImpl.class);
+
+    private final int dbconnectionErrorTolerance;
+    private final boolean dbconnectionFence;
+    private final ImmutableMap<Integer, String> fatalErrors = ImmutableMap.<Integer, String>builder()
+            .put(1129, "Management Server is blocked because of many connection errors; unblock with mysqladmin flush-hosts")
+            .build();
+    private static AtomicInteger dbconnectionErrorCount = new AtomicInteger();
+    private static final int DEFAULT_CONNECTION_ERROR_TOLERANCE=100;
+
+    public static void setDbconnectionErrorCount(AtomicInteger dbconnectionErrorCount) {
+        DbConnectionObserverImpl.dbconnectionErrorCount = dbconnectionErrorCount;
+    }
+
+    public DbConnectionObserverImpl() {
+        Properties dbProps = DbProperties.getDbProperties();
+        dbconnectionErrorTolerance = NumberUtils.toInt(dbProps.getProperty("db.connection.error.tolerance"), DEFAULT_CONNECTION_ERROR_TOLERANCE);
+        if (dbconnectionErrorTolerance < 0){
+            throw new IllegalStateException("Negative value for db.connection.error.tolerance in db.properties, value=" + dbconnectionErrorTolerance);
+        }
+        dbconnectionFence = Boolean.parseBoolean(dbProps.getProperty("db.connection.errors.fence"));
+        s_logger.info(this);
+    }
+
+    @Override
+    public void onError(SQLException se) {
+        if (dbconnectionFence) {
+            String desc = isFatal(se);
+            if (desc != null) {
+                s_logger.error("Fencing management server due to fatal exception " + desc, se);
+                System.exit(1);
+            } else if (dbconnectionErrorCount.intValue() > dbconnectionErrorTolerance) {
+                s_logger.error("Fencing management server due to database connection errors " + se.getMessage(), se);
+                System.exit(1);
+            }
+            dbconnectionErrorCount.incrementAndGet();
+        }
+    }
+
+    private String isFatal(SQLException se) {
+        Integer error_code = se.getErrorCode();
+        return fatalErrors.get(error_code);
+    }
+
+    @Override
+    public void onSuccess() {
+        dbconnectionErrorCount.set(0);
+    }
+
+    @Override
+    public String toString() {
+        return "DbConnectionObserverImpl [dbconnectionErrorTolerance=" + dbconnectionErrorTolerance + ", dbconnectionFence=" + dbconnectionFence + ", fatalErrors=" + fatalErrors
+                + "]";
+    }
+
+}

framework/db/src/com/cloud/utils/db/TransactionLegacy.java

@@ -67,6 +67,7 @@ public class TransactionLegacy implements Closeable {
     private static final Logger s_lockLogger = Logger.getLogger(Transaction.class.getName() + "." + "Lock");
     private static final Logger s_connLogger = Logger.getLogger(Transaction.class.getName() + "." + "Connection");
 
+
     private static final ThreadLocal<TransactionLegacy> tls = new ThreadLocal<TransactionLegacy>();
     private static final String START_TXN = "start_txn";
     private static final String CURRENT_TXN = "current_txn";
@@ -83,6 +84,8 @@ public class TransactionLegacy implements Closeable {
     public static final short CONNECTED_DB = -1;
 
     private static AtomicLong s_id = new AtomicLong();
+
+    private static final DbConnectionObserver dbConnectionObserver = new DbConnectionObserverImpl();
     private static final TransactionMBeanImpl s_mbean = new TransactionMBeanImpl();
     static {
         try {
@@ -557,7 +560,14 @@ public class TransactionLegacy implements Closeable {
             switch (_dbId) {
             case CLOUD_DB:
                 if (s_ds != null) {
-                    _conn = s_ds.getConnection();
+                    try {
+                        _conn = s_ds.getConnection();
+                        dbConnectionObserver.onSuccess();
+                    }
+                    catch (SQLException se){
+                        dbConnectionObserver.onError(se);
+                        throw  se;
+                    }
                 } else {
                     s_logger.warn("A static-initialized variable becomes null, process is dying?");
                     throw new CloudRuntimeException("Database is not initialized, process is dying?");

server/resources/META-INF/cloudstack/system/spring-server-system-context.xml

@@ -33,4 +33,5 @@
         </constructor-arg>
     </bean>
 
+
 </beans>

utils/conf/db.properties

@@ -31,6 +31,9 @@ db.root.password=
 db.cloud.host=localhost
 db.cloud.port=3306
 db.cloud.name=cloud
+db.connection.errors.fence=true
+db.connection.error.tolerance=3
+
 
 # CloudStack database tuning parameters
 db.cloud.maxActive=250