From 6fdd19fa7e82e5edac9c28ea3900bdd5a7be2ab6 Mon Sep 17 00:00:00 2001 From: Abhinandan Prateek Date: Thu, 27 Oct 2016 16:04:54 +0530 Subject: [PATCH] CLOUDSTACK-9571: Fence DB if there are consecutive connection errors. --- client/tomcatconf/db.properties.in | 2 + .../spring-framework-db-system-context.xml | 15 ++-- .../cloud/utils/db/DbConnectionObserver.java | 8 +++ .../utils/db/DbConnectionObserverImpl.java | 68 +++++++++++++++++++ .../com/cloud/utils/db/TransactionLegacy.java | 12 +++- .../system/spring-server-system-context.xml | 1 + utils/conf/db.properties | 3 + 7 files changed, 100 insertions(+), 9 deletions(-) create mode 100644 framework/db/src/com/cloud/utils/db/DbConnectionObserver.java create mode 100644 framework/db/src/com/cloud/utils/db/DbConnectionObserverImpl.java diff --git a/client/tomcatconf/db.properties.in b/client/tomcatconf/db.properties.in index 8f6980b7db1..b4a93875783 100644 --- a/client/tomcatconf/db.properties.in +++ b/client/tomcatconf/db.properties.in @@ -27,6 +27,8 @@ db.cloud.password=@DBPW@ db.cloud.host=@DBHOST@ db.cloud.port=3306 db.cloud.name=cloud +db.connection.errors.fence=true +db.connection.error.tolerance=2 # CloudStack database tuning parameters db.cloud.maxActive=250 diff --git a/framework/db/resources/META-INF/cloudstack/system/spring-framework-db-system-context.xml b/framework/db/resources/META-INF/cloudstack/system/spring-framework-db-system-context.xml index 651b8742a33..2570d985e26 100644 --- a/framework/db/resources/META-INF/cloudstack/system/spring-framework-db-system-context.xml +++ b/framework/db/resources/META-INF/cloudstack/system/spring-framework-db-system-context.xml @@ -17,16 +17,15 @@ under the License. --> + http://www.springframework.org/schema/context/spring-context-3.0.xsd"> + + - - diff --git a/framework/db/src/com/cloud/utils/db/DbConnectionObserver.java b/framework/db/src/com/cloud/utils/db/DbConnectionObserver.java new file mode 100644 index 00000000000..6ce853ff364 --- /dev/null +++ b/framework/db/src/com/cloud/utils/db/DbConnectionObserver.java @@ -0,0 +1,8 @@ +package com.cloud.utils.db; + +import java.sql.SQLException; + +public interface DbConnectionObserver { + void onError(SQLException se); + void onSuccess(); +} diff --git a/framework/db/src/com/cloud/utils/db/DbConnectionObserverImpl.java b/framework/db/src/com/cloud/utils/db/DbConnectionObserverImpl.java new file mode 100644 index 00000000000..0b44d149e1e --- /dev/null +++ b/framework/db/src/com/cloud/utils/db/DbConnectionObserverImpl.java @@ -0,0 +1,68 @@ +package com.cloud.utils.db; + +import java.sql.SQLException; +import java.util.Properties; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.commons.lang.math.NumberUtils; +import org.apache.log4j.Logger; + +import com.google.common.collect.ImmutableMap; + +public class DbConnectionObserverImpl implements DbConnectionObserver { + private static final Logger s_logger = Logger.getLogger(DbConnectionObserverImpl.class); + + private final int dbconnectionErrorTolerance; + private final boolean dbconnectionFence; + private final ImmutableMap fatalErrors = ImmutableMap.builder() + .put(1129, "Management Server is blocked because of many connection errors; unblock with mysqladmin flush-hosts") + .build(); + private static AtomicInteger dbconnectionErrorCount = new AtomicInteger(); + private static final int DEFAULT_CONNECTION_ERROR_TOLERANCE=100; + + public static void setDbconnectionErrorCount(AtomicInteger dbconnectionErrorCount) { + DbConnectionObserverImpl.dbconnectionErrorCount = dbconnectionErrorCount; + } + + public DbConnectionObserverImpl() { + Properties dbProps = DbProperties.getDbProperties(); + dbconnectionErrorTolerance = NumberUtils.toInt(dbProps.getProperty("db.connection.error.tolerance"), DEFAULT_CONNECTION_ERROR_TOLERANCE); + if (dbconnectionErrorTolerance < 0){ + throw new IllegalStateException("Negative value for db.connection.error.tolerance in db.properties, value=" + dbconnectionErrorTolerance); + } + dbconnectionFence = Boolean.parseBoolean(dbProps.getProperty("db.connection.errors.fence")); + s_logger.info(this); + } + + @Override + public void onError(SQLException se) { + if (dbconnectionFence) { + String desc = isFatal(se); + if (desc != null) { + s_logger.error("Fencing management server due to fatal exception " + desc, se); + System.exit(1); + } else if (dbconnectionErrorCount.intValue() > dbconnectionErrorTolerance) { + s_logger.error("Fencing management server due to database connection errors " + se.getMessage(), se); + System.exit(1); + } + dbconnectionErrorCount.incrementAndGet(); + } + } + + private String isFatal(SQLException se) { + Integer error_code = se.getErrorCode(); + return fatalErrors.get(error_code); + } + + @Override + public void onSuccess() { + dbconnectionErrorCount.set(0); + } + + @Override + public String toString() { + return "DbConnectionObserverImpl [dbconnectionErrorTolerance=" + dbconnectionErrorTolerance + ", dbconnectionFence=" + dbconnectionFence + ", fatalErrors=" + fatalErrors + + "]"; + } + +} diff --git a/framework/db/src/com/cloud/utils/db/TransactionLegacy.java b/framework/db/src/com/cloud/utils/db/TransactionLegacy.java index 7784d2f4399..8947fbd1b66 100755 --- a/framework/db/src/com/cloud/utils/db/TransactionLegacy.java +++ b/framework/db/src/com/cloud/utils/db/TransactionLegacy.java @@ -67,6 +67,7 @@ public class TransactionLegacy implements Closeable { private static final Logger s_lockLogger = Logger.getLogger(Transaction.class.getName() + "." + "Lock"); private static final Logger s_connLogger = Logger.getLogger(Transaction.class.getName() + "." + "Connection"); + private static final ThreadLocal tls = new ThreadLocal(); private static final String START_TXN = "start_txn"; private static final String CURRENT_TXN = "current_txn"; @@ -83,6 +84,8 @@ public class TransactionLegacy implements Closeable { public static final short CONNECTED_DB = -1; private static AtomicLong s_id = new AtomicLong(); + + private static final DbConnectionObserver dbConnectionObserver = new DbConnectionObserverImpl(); private static final TransactionMBeanImpl s_mbean = new TransactionMBeanImpl(); static { try { @@ -557,7 +560,14 @@ public class TransactionLegacy implements Closeable { switch (_dbId) { case CLOUD_DB: if (s_ds != null) { - _conn = s_ds.getConnection(); + try { + _conn = s_ds.getConnection(); + dbConnectionObserver.onSuccess(); + } + catch (SQLException se){ + dbConnectionObserver.onError(se); + throw se; + } } else { s_logger.warn("A static-initialized variable becomes null, process is dying?"); throw new CloudRuntimeException("Database is not initialized, process is dying?"); diff --git a/server/resources/META-INF/cloudstack/system/spring-server-system-context.xml b/server/resources/META-INF/cloudstack/system/spring-server-system-context.xml index a14a0738413..f01b8656752 100644 --- a/server/resources/META-INF/cloudstack/system/spring-server-system-context.xml +++ b/server/resources/META-INF/cloudstack/system/spring-server-system-context.xml @@ -33,4 +33,5 @@ + \ No newline at end of file diff --git a/utils/conf/db.properties b/utils/conf/db.properties index cdd29095db7..7e775d39cb9 100644 --- a/utils/conf/db.properties +++ b/utils/conf/db.properties @@ -31,6 +31,9 @@ db.root.password= db.cloud.host=localhost db.cloud.port=3306 db.cloud.name=cloud +db.connection.errors.fence=true +db.connection.error.tolerance=3 + # CloudStack database tuning parameters db.cloud.maxActive=250