diff --git a/client/tomcatconf/db.properties.in b/client/tomcatconf/db.properties.in
index 8f6980b7db1..b4a93875783 100644
--- a/client/tomcatconf/db.properties.in
+++ b/client/tomcatconf/db.properties.in
@@ -27,6 +27,8 @@ db.cloud.password=@DBPW@
db.cloud.host=@DBHOST@
db.cloud.port=3306
db.cloud.name=cloud
+db.connection.errors.fence=true
+db.connection.error.tolerance=2
# CloudStack database tuning parameters
db.cloud.maxActive=250
diff --git a/framework/db/resources/META-INF/cloudstack/system/spring-framework-db-system-context.xml b/framework/db/resources/META-INF/cloudstack/system/spring-framework-db-system-context.xml
index 651b8742a33..2570d985e26 100644
--- a/framework/db/resources/META-INF/cloudstack/system/spring-framework-db-system-context.xml
+++ b/framework/db/resources/META-INF/cloudstack/system/spring-framework-db-system-context.xml
@@ -17,16 +17,15 @@
under the License.
-->
+ http://www.springframework.org/schema/context/spring-context-3.0.xsd">
+
+
-
-
diff --git a/framework/db/src/com/cloud/utils/db/DbConnectionObserver.java b/framework/db/src/com/cloud/utils/db/DbConnectionObserver.java
new file mode 100644
index 00000000000..6ce853ff364
--- /dev/null
+++ b/framework/db/src/com/cloud/utils/db/DbConnectionObserver.java
@@ -0,0 +1,8 @@
+package com.cloud.utils.db;
+
+import java.sql.SQLException;
+
+public interface DbConnectionObserver {
+ void onError(SQLException se);
+ void onSuccess();
+}
diff --git a/framework/db/src/com/cloud/utils/db/DbConnectionObserverImpl.java b/framework/db/src/com/cloud/utils/db/DbConnectionObserverImpl.java
new file mode 100644
index 00000000000..0b44d149e1e
--- /dev/null
+++ b/framework/db/src/com/cloud/utils/db/DbConnectionObserverImpl.java
@@ -0,0 +1,68 @@
+package com.cloud.utils.db;
+
+import java.sql.SQLException;
+import java.util.Properties;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.commons.lang.math.NumberUtils;
+import org.apache.log4j.Logger;
+
+import com.google.common.collect.ImmutableMap;
+
+public class DbConnectionObserverImpl implements DbConnectionObserver {
+ private static final Logger s_logger = Logger.getLogger(DbConnectionObserverImpl.class);
+
+ private final int dbconnectionErrorTolerance;
+ private final boolean dbconnectionFence;
+ private final ImmutableMap fatalErrors = ImmutableMap.builder()
+ .put(1129, "Management Server is blocked because of many connection errors; unblock with mysqladmin flush-hosts")
+ .build();
+ private static AtomicInteger dbconnectionErrorCount = new AtomicInteger();
+ private static final int DEFAULT_CONNECTION_ERROR_TOLERANCE=100;
+
+ public static void setDbconnectionErrorCount(AtomicInteger dbconnectionErrorCount) {
+ DbConnectionObserverImpl.dbconnectionErrorCount = dbconnectionErrorCount;
+ }
+
+ public DbConnectionObserverImpl() {
+ Properties dbProps = DbProperties.getDbProperties();
+ dbconnectionErrorTolerance = NumberUtils.toInt(dbProps.getProperty("db.connection.error.tolerance"), DEFAULT_CONNECTION_ERROR_TOLERANCE);
+ if (dbconnectionErrorTolerance < 0){
+ throw new IllegalStateException("Negative value for db.connection.error.tolerance in db.properties, value=" + dbconnectionErrorTolerance);
+ }
+ dbconnectionFence = Boolean.parseBoolean(dbProps.getProperty("db.connection.errors.fence"));
+ s_logger.info(this);
+ }
+
+ @Override
+ public void onError(SQLException se) {
+ if (dbconnectionFence) {
+ String desc = isFatal(se);
+ if (desc != null) {
+ s_logger.error("Fencing management server due to fatal exception " + desc, se);
+ System.exit(1);
+ } else if (dbconnectionErrorCount.intValue() > dbconnectionErrorTolerance) {
+ s_logger.error("Fencing management server due to database connection errors " + se.getMessage(), se);
+ System.exit(1);
+ }
+ dbconnectionErrorCount.incrementAndGet();
+ }
+ }
+
+ private String isFatal(SQLException se) {
+ Integer error_code = se.getErrorCode();
+ return fatalErrors.get(error_code);
+ }
+
+ @Override
+ public void onSuccess() {
+ dbconnectionErrorCount.set(0);
+ }
+
+ @Override
+ public String toString() {
+ return "DbConnectionObserverImpl [dbconnectionErrorTolerance=" + dbconnectionErrorTolerance + ", dbconnectionFence=" + dbconnectionFence + ", fatalErrors=" + fatalErrors
+ + "]";
+ }
+
+}
diff --git a/framework/db/src/com/cloud/utils/db/TransactionLegacy.java b/framework/db/src/com/cloud/utils/db/TransactionLegacy.java
index 7784d2f4399..8947fbd1b66 100755
--- a/framework/db/src/com/cloud/utils/db/TransactionLegacy.java
+++ b/framework/db/src/com/cloud/utils/db/TransactionLegacy.java
@@ -67,6 +67,7 @@ public class TransactionLegacy implements Closeable {
private static final Logger s_lockLogger = Logger.getLogger(Transaction.class.getName() + "." + "Lock");
private static final Logger s_connLogger = Logger.getLogger(Transaction.class.getName() + "." + "Connection");
+
private static final ThreadLocal tls = new ThreadLocal();
private static final String START_TXN = "start_txn";
private static final String CURRENT_TXN = "current_txn";
@@ -83,6 +84,8 @@ public class TransactionLegacy implements Closeable {
public static final short CONNECTED_DB = -1;
private static AtomicLong s_id = new AtomicLong();
+
+ private static final DbConnectionObserver dbConnectionObserver = new DbConnectionObserverImpl();
private static final TransactionMBeanImpl s_mbean = new TransactionMBeanImpl();
static {
try {
@@ -557,7 +560,14 @@ public class TransactionLegacy implements Closeable {
switch (_dbId) {
case CLOUD_DB:
if (s_ds != null) {
- _conn = s_ds.getConnection();
+ try {
+ _conn = s_ds.getConnection();
+ dbConnectionObserver.onSuccess();
+ }
+ catch (SQLException se){
+ dbConnectionObserver.onError(se);
+ throw se;
+ }
} else {
s_logger.warn("A static-initialized variable becomes null, process is dying?");
throw new CloudRuntimeException("Database is not initialized, process is dying?");
diff --git a/server/resources/META-INF/cloudstack/system/spring-server-system-context.xml b/server/resources/META-INF/cloudstack/system/spring-server-system-context.xml
index a14a0738413..f01b8656752 100644
--- a/server/resources/META-INF/cloudstack/system/spring-server-system-context.xml
+++ b/server/resources/META-INF/cloudstack/system/spring-server-system-context.xml
@@ -33,4 +33,5 @@
+
\ No newline at end of file
diff --git a/utils/conf/db.properties b/utils/conf/db.properties
index cdd29095db7..7e775d39cb9 100644
--- a/utils/conf/db.properties
+++ b/utils/conf/db.properties
@@ -31,6 +31,9 @@ db.root.password=
db.cloud.host=localhost
db.cloud.port=3306
db.cloud.name=cloud
+db.connection.errors.fence=true
+db.connection.error.tolerance=3
+
# CloudStack database tuning parameters
db.cloud.maxActive=250