From dcd3c44279478b237f8fdcb6f8f9e301b9ae3195 Mon Sep 17 00:00:00 2001 From: Damodar Reddy Date: Fri, 8 Nov 2013 14:03:03 +0530 Subject: [PATCH] CLOUDSTACK-4916: To Enable Master High Availability using mysql connector params and mysql's 2 way replication Signed-off-by: Abhinandan Prateek --- client/tomcatconf/db.properties.in | 24 ++++++++++- .../cloud/utils/db/ConnectionConcierge.java | 6 ++- .../src/com/cloud/utils/db/Merovingian2.java | 10 +++-- .../com/cloud/utils/db/TransactionLegacy.java | 43 +++++++++++++------ 4 files changed, 63 insertions(+), 20 deletions(-) diff --git a/client/tomcatconf/db.properties.in b/client/tomcatconf/db.properties.in index b0cb22b26f0..50b8d334136 100644 --- a/client/tomcatconf/db.properties.in +++ b/client/tomcatconf/db.properties.in @@ -32,7 +32,6 @@ db.cloud.name=cloud db.cloud.maxActive=250 db.cloud.maxIdle=30 db.cloud.maxWait=10000 -db.cloud.autoReconnect=true db.cloud.validationQuery=SELECT 1 db.cloud.testOnBorrow=true db.cloud.testWhileIdle=true @@ -63,7 +62,6 @@ db.usage.name=cloud_usage db.usage.maxActive=100 db.usage.maxIdle=30 db.usage.maxWait=10000 -db.usage.autoReconnect=true db.usage.url.params= # awsapi database settings @@ -84,3 +82,25 @@ db.simulator.maxIdle=30 db.simulator.maxWait=10000 db.simulator.autoReconnect=true + +# High Availability And Cluster Properties +db.ha.enabled=false +# cloud stack Database +db.cloud.slaves=localhost,localhost #Comma Separated list of slaves +db.cloud.autoReconnect=true +db.cloud.failOverReadOnly=false #Do not change this parameter to true +db.cloud.reconnectAtTxEnd=true +db.cloud.autoReconnectForPools=true +db.cloud.secondsBeforeRetryMaster=3600 +db.cloud.queriesBeforeRetryMaster=5000 +db.cloud.initialTimeout=3600 + +#usage Database +db.usage.slaves=localhost,localhost #Comma Separated list of slaves +db.usage.autoReconnect=true +db.usage.failOverReadOnly=false #Do not change this parameter to true +db.usage.reconnectAtTxEnd=true +db.usage.autoReconnectForPools=true +db.usage.secondsBeforeRetryMaster=3600 +db.usage.queriesBeforeRetryMaster=5000 +db.usage.initialTimeout=3600 diff --git a/framework/db/src/com/cloud/utils/db/ConnectionConcierge.java b/framework/db/src/com/cloud/utils/db/ConnectionConcierge.java index acb9cc6851b..bb76d0f03c8 100644 --- a/framework/db/src/com/cloud/utils/db/ConnectionConcierge.java +++ b/framework/db/src/com/cloud/utils/db/ConnectionConcierge.java @@ -144,8 +144,10 @@ public class ConnectionConcierge { PreparedStatement pstmt = null; try { if (conn != null) { - pstmt = conn.prepareStatement("SELECT 1"); - pstmt.executeQuery(); + synchronized (conn) { + pstmt = conn.prepareStatement("SELECT 1"); + pstmt.executeQuery(); + } } return null; } catch (Throwable th) { diff --git a/framework/db/src/com/cloud/utils/db/Merovingian2.java b/framework/db/src/com/cloud/utils/db/Merovingian2.java index 8544aab0384..15e3d24299f 100644 --- a/framework/db/src/com/cloud/utils/db/Merovingian2.java +++ b/framework/db/src/com/cloud/utils/db/Merovingian2.java @@ -253,10 +253,12 @@ public class Merovingian2 extends StandardMBean implements MerovingianMBean { s_logger.info("Cleaning up locks for " + msId); PreparedStatement pstmt = null; try { - pstmt = _concierge.conn().prepareStatement(CLEANUP_MGMT_LOCKS_SQL); - pstmt.setLong(1, msId); - int rows = pstmt.executeUpdate(); - s_logger.info("Released " + rows + " locks for " + msId); + synchronized (_concierge.conn()) { + pstmt = _concierge.conn().prepareStatement(CLEANUP_MGMT_LOCKS_SQL); + pstmt.setLong(1, msId); + int rows = pstmt.executeUpdate(); + s_logger.info("Released " + rows + " locks for " + msId); + } } catch (SQLException e) { throw new CloudRuntimeException("Unable to clear the locks", e); } finally { diff --git a/framework/db/src/com/cloud/utils/db/TransactionLegacy.java b/framework/db/src/com/cloud/utils/db/TransactionLegacy.java index a874ce10a56..39893bd4eab 100755 --- a/framework/db/src/com/cloud/utils/db/TransactionLegacy.java +++ b/framework/db/src/com/cloud/utils/db/TransactionLegacy.java @@ -17,7 +17,6 @@ package com.cloud.utils.db; import java.io.File; -import java.io.FileInputStream; import java.io.IOException; import java.sql.Connection; import java.sql.PreparedStatement; @@ -41,12 +40,9 @@ import org.apache.commons.pool.KeyedObjectPoolFactory; import org.apache.commons.pool.impl.GenericObjectPool; import org.apache.commons.pool.impl.StackKeyedObjectPoolFactory; import org.apache.log4j.Logger; -import org.jasypt.encryption.pbe.StandardPBEStringEncryptor; -import org.jasypt.properties.EncryptableProperties; import com.cloud.utils.Pair; import com.cloud.utils.PropertiesUtil; -import com.cloud.utils.crypt.EncryptionSecretKeyChecker; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.mgmt.JmxUtil; @@ -108,8 +104,6 @@ public class TransactionLegacy { private Statement _stmt; private String _creator; - private TransactionLegacy _prev = null; - public static TransactionLegacy currentTxn() { return currentTxn(true); } @@ -1016,6 +1010,7 @@ public class TransactionLegacy { private static DataSource s_usageDS; private static DataSource s_awsapiDS; private static DataSource s_simulatorDS; + private static boolean s_dbHAEnabled; static { // Initialize with assumed db.properties file @@ -1031,11 +1026,15 @@ public class TransactionLegacy { } } + @SuppressWarnings({ "rawtypes", "unchecked" }) public static void initDataSource(Properties dbProps) { try { if (dbProps.size() == 0) return; + s_dbHAEnabled = Boolean.valueOf(dbProps.getProperty("db.ha.enabled")); + s_logger.info("Is Data Base High Availiability enabled? Ans : " + s_dbHAEnabled); + // FIXME: If params are missing...default them???? final int cloudMaxActive = Integer.parseInt(dbProps.getProperty("db.cloud.maxActive")); final int cloudMaxIdle = Integer.parseInt(dbProps.getProperty("db.cloud.maxIdle")); @@ -1071,6 +1070,14 @@ public class TransactionLegacy { final boolean cloudPoolPreparedStatements = Boolean.parseBoolean(dbProps.getProperty("db.cloud.poolPreparedStatements")); final String url = dbProps.getProperty("db.cloud.url.params"); + String cloudDbHAParams = null; + String cloudSlaves = null; + if(s_dbHAEnabled) { + cloudDbHAParams = getDBHAParams("cloud", dbProps); + cloudSlaves = dbProps.getProperty("db.cloud.slaves"); + s_logger.info("The slaves configured for Cloud Data base is/are : " + cloudSlaves); + } + final boolean useSSL = Boolean.parseBoolean(dbProps.getProperty("db.cloud.useSSL")); if (useSSL) { System.setProperty("javax.net.ssl.keyStore", dbProps.getProperty("db.cloud.keyStore")); @@ -1082,8 +1089,8 @@ public class TransactionLegacy { final GenericObjectPool cloudConnectionPool = new GenericObjectPool(null, cloudMaxActive, GenericObjectPool.DEFAULT_WHEN_EXHAUSTED_ACTION, cloudMaxWait, cloudMaxIdle, cloudTestOnBorrow, false, cloudTimeBtwEvictionRunsMillis, 1, cloudMinEvcitableIdleTimeMillis, cloudTestWhileIdle); - final ConnectionFactory cloudConnectionFactory = new DriverManagerConnectionFactory("jdbc:mysql://" + cloudHost + ":" + cloudPort + "/" + cloudDbName + - "?autoReconnect=" + cloudAutoReconnect + (url != null ? "&" + url : "") + (useSSL ? "&useSSL=true" : ""), cloudUsername, cloudPassword); + final ConnectionFactory cloudConnectionFactory = new DriverManagerConnectionFactory("jdbc:mysql://" + cloudHost + (s_dbHAEnabled ? "," + cloudSlaves : "") + ":" + cloudPort + "/" + cloudDbName + + "?autoReconnect=" + cloudAutoReconnect + (url != null ? "&" + url : "") + (useSSL ? "&useSSL=true" : "") + (s_dbHAEnabled ? "&" + cloudDbHAParams : ""), cloudUsername, cloudPassword); final KeyedObjectPoolFactory poolableObjFactory = (cloudPoolPreparedStatements ? new StackKeyedObjectPoolFactory() : null); @@ -1108,8 +1115,8 @@ public class TransactionLegacy { final GenericObjectPool usageConnectionPool = new GenericObjectPool(null, usageMaxActive, GenericObjectPool.DEFAULT_WHEN_EXHAUSTED_ACTION, usageMaxWait, usageMaxIdle); - final ConnectionFactory usageConnectionFactory = new DriverManagerConnectionFactory("jdbc:mysql://" + usageHost + ":" + usagePort + "/" + usageDbName + - "?autoReconnect=" + usageAutoReconnect + (usageUrl != null ? "&" + usageUrl : ""), usageUsername, usagePassword); + final ConnectionFactory usageConnectionFactory = new DriverManagerConnectionFactory("jdbc:mysql://" + usageHost + (s_dbHAEnabled ? "," + dbProps.getProperty("db.cloud.slaves") : "") + ":" + usagePort + "/" + usageDbName + + "?autoReconnect=" + usageAutoReconnect + (usageUrl != null ? "&" + usageUrl : "") + (s_dbHAEnabled ? "&" + getDBHAParams("usage", dbProps) : ""), usageUsername, usagePassword); final PoolableConnectionFactory usagePoolableConnectionFactory = new PoolableConnectionFactory(usageConnectionFactory, usageConnectionPool, new StackKeyedObjectPoolFactory(), null, false, false); @@ -1121,8 +1128,8 @@ public class TransactionLegacy { final String awsapiDbName = dbProps.getProperty("db.awsapi.name"); final GenericObjectPool awsapiConnectionPool = new GenericObjectPool(null, usageMaxActive, GenericObjectPool.DEFAULT_WHEN_EXHAUSTED_ACTION, usageMaxWait, usageMaxIdle); - final ConnectionFactory awsapiConnectionFactory = new DriverManagerConnectionFactory("jdbc:mysql://" + cloudHost + ":" + cloudPort + "/" + awsapiDbName + - "?autoReconnect=" + usageAutoReconnect, cloudUsername, cloudPassword); + final ConnectionFactory awsapiConnectionFactory = new DriverManagerConnectionFactory("jdbc:mysql://" + cloudHost + (s_dbHAEnabled ? "," + cloudSlaves : "") + ":" + cloudPort + "/" + awsapiDbName + + "?autoReconnect=" + cloudAutoReconnect + (s_dbHAEnabled ? "&" + cloudDbHAParams : ""), cloudUsername, cloudPassword); final PoolableConnectionFactory awsapiPoolableConnectionFactory = new PoolableConnectionFactory(awsapiConnectionFactory, awsapiConnectionPool, new StackKeyedObjectPoolFactory(), null, false, false); @@ -1160,7 +1167,19 @@ public class TransactionLegacy { s_logger.warn("Unable to load db configuration, using defaults with 5 connections. Falling back on assumed datasource on localhost:3306 using username:password=cloud:cloud. Please check your configuration", e); } } + + private static String getDBHAParams(String dbName,Properties dbProps) { + StringBuilder sb = new StringBuilder(); + sb.append("failOverReadOnly=" + dbProps.getProperty("db." + dbName + ".failOverReadOnly")); + sb.append("&").append("reconnectAtTxEnd=" + dbProps.getProperty("db." + dbName + ".reconnectAtTxEnd")); + sb.append("&").append("autoReconnectForPools=" + dbProps.getProperty("db." + dbName + ".autoReconnectForPools")); + sb.append("&").append("secondsBeforeRetryMaster=" + dbProps.getProperty("db." + dbName + ".secondsBeforeRetryMaster")); + sb.append("&").append("queriesBeforeRetryMaster=" + dbProps.getProperty("db." + dbName + ".queriesBeforeRetryMaster")); + sb.append("&").append("initialTimeout=" + dbProps.getProperty("db." + dbName + ".initialTimeout")); + return sb.toString(); + } + @SuppressWarnings({ "unchecked", "rawtypes" }) private static DataSource getDefaultDataSource(final String database) { final GenericObjectPool connectionPool = new GenericObjectPool(null, 5); final ConnectionFactory connectionFactory = new DriverManagerConnectionFactory(