From 67fc2f53e2e197f2a8da3b7fc367de9dbe54bb93 Mon Sep 17 00:00:00 2001 From: Damodar Reddy Date: Thu, 14 Nov 2013 10:45:54 +0530 Subject: [PATCH] CLOUDSTACK-4916: Fixing the issue with DB HA when there are more than 2 db nodes (Issue was mysql DriverManager was returning salve db nodes on random basis which was causing issue) Signed-off-by: Abhinandan Prateek --- client/tomcatconf/db.properties.in | 1 + framework/db/pom.xml | 5 + .../com/cloud/utils/db/StaticStrategy.java | 130 ++++++++++++++++++ .../com/cloud/utils/db/TransactionLegacy.java | 8 +- 4 files changed, 140 insertions(+), 4 deletions(-) create mode 100644 framework/db/src/com/cloud/utils/db/StaticStrategy.java diff --git a/client/tomcatconf/db.properties.in b/client/tomcatconf/db.properties.in index 31e08033370..b224cec9700 100644 --- a/client/tomcatconf/db.properties.in +++ b/client/tomcatconf/db.properties.in @@ -85,6 +85,7 @@ db.simulator.autoReconnect=true # High Availability And Cluster Properties db.ha.enabled=false +db.ha.loadBalanceStrategy=com.cloud.utils.db.StaticStrategy # cloud stack Database db.cloud.slaves=localhost,localhost db.cloud.autoReconnect=true diff --git a/framework/db/pom.xml b/framework/db/pom.xml index 5af00a00747..e28628e5e9f 100644 --- a/framework/db/pom.xml +++ b/framework/db/pom.xml @@ -44,6 +44,11 @@ cloud-utils ${project.version} + + mysql + mysql-connector-java + compile + diff --git a/framework/db/src/com/cloud/utils/db/StaticStrategy.java b/framework/db/src/com/cloud/utils/db/StaticStrategy.java new file mode 100644 index 00000000000..29e96dfc18e --- /dev/null +++ b/framework/db/src/com/cloud/utils/db/StaticStrategy.java @@ -0,0 +1,130 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.utils.db; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import com.mysql.jdbc.BalanceStrategy; +import com.mysql.jdbc.Connection; +import com.mysql.jdbc.ConnectionImpl; +import com.mysql.jdbc.LoadBalancingConnectionProxy; +import com.mysql.jdbc.SQLError; + +public class StaticStrategy implements BalanceStrategy { + + public StaticStrategy() { + } + + public void destroy() { + // we don't have anything to clean up + } + + public void init(Connection conn, Properties props) throws SQLException { + // we don't have anything to initialize + } + + public ConnectionImpl pickConnection(LoadBalancingConnectionProxy proxy, + List configuredHosts, Map liveConnections, long[] responseTimes, + int numRetries) throws SQLException { + int numHosts = configuredHosts.size(); + + SQLException ex = null; + + List whiteList = new ArrayList(numHosts); + whiteList.addAll(configuredHosts); + + Map blackList = proxy.getGlobalBlacklist(); + + whiteList.removeAll(blackList.keySet()); + + Map whiteListMap = this.getArrayIndexMap(whiteList); + + + for (int attempts = 0; attempts < numRetries;) { + if(whiteList.size() == 0){ + throw SQLError.createSQLException("No hosts configured", null); + } + + String hostPortSpec = whiteList.get(0); //Always take the first host + + ConnectionImpl conn = liveConnections.get(hostPortSpec); + + if (conn == null) { + try { + conn = proxy.createConnectionForHost(hostPortSpec); + } catch (SQLException sqlEx) { + ex = sqlEx; + + if (proxy.shouldExceptionTriggerFailover(sqlEx)) { + + Integer whiteListIndex = whiteListMap.get(hostPortSpec); + + // exclude this host from being picked again + if (whiteListIndex != null) { + whiteList.remove(whiteListIndex.intValue()); + whiteListMap = this.getArrayIndexMap(whiteList); + } + proxy.addToGlobalBlacklist( hostPortSpec ); + + if (whiteList.size() == 0) { + attempts++; + try { + Thread.sleep(250); + } catch (InterruptedException e) { + } + + // start fresh + whiteListMap = new HashMap(numHosts); + whiteList.addAll(configuredHosts); + blackList = proxy.getGlobalBlacklist(); + + whiteList.removeAll(blackList.keySet()); + whiteListMap = this.getArrayIndexMap(whiteList); + } + + continue; + } + + throw sqlEx; + } + } + + return conn; + } + + if (ex != null) { + throw ex; + } + + return null; // we won't get here, compiler can't tell + } + + private Map getArrayIndexMap(List l) { + Map m = new HashMap(l.size()); + for (int i = 0; i < l.size(); i++) { + m.put(l.get(i), Integer.valueOf(i)); + } + return m; + + } + +} \ No newline at end of file diff --git a/framework/db/src/com/cloud/utils/db/TransactionLegacy.java b/framework/db/src/com/cloud/utils/db/TransactionLegacy.java index 39893bd4eab..d50118bdc8f 100755 --- a/framework/db/src/com/cloud/utils/db/TransactionLegacy.java +++ b/framework/db/src/com/cloud/utils/db/TransactionLegacy.java @@ -1034,7 +1034,7 @@ public class TransactionLegacy { s_dbHAEnabled = Boolean.valueOf(dbProps.getProperty("db.ha.enabled")); s_logger.info("Is Data Base High Availiability enabled? Ans : " + s_dbHAEnabled); - + String loadBalanceStrategy = dbProps.getProperty("db.ha.loadBalanceStrategy"); // FIXME: If params are missing...default them???? final int cloudMaxActive = Integer.parseInt(dbProps.getProperty("db.cloud.maxActive")); final int cloudMaxIdle = Integer.parseInt(dbProps.getProperty("db.cloud.maxIdle")); @@ -1090,7 +1090,7 @@ public class TransactionLegacy { cloudMaxWait, cloudMaxIdle, cloudTestOnBorrow, false, cloudTimeBtwEvictionRunsMillis, 1, cloudMinEvcitableIdleTimeMillis, cloudTestWhileIdle); final ConnectionFactory cloudConnectionFactory = new DriverManagerConnectionFactory("jdbc:mysql://" + cloudHost + (s_dbHAEnabled ? "," + cloudSlaves : "") + ":" + cloudPort + "/" + cloudDbName + - "?autoReconnect=" + cloudAutoReconnect + (url != null ? "&" + url : "") + (useSSL ? "&useSSL=true" : "") + (s_dbHAEnabled ? "&" + cloudDbHAParams : ""), cloudUsername, cloudPassword); + "?autoReconnect=" + cloudAutoReconnect + (url != null ? "&" + url : "") + (useSSL ? "&useSSL=true" : "") + (s_dbHAEnabled ? "&" + cloudDbHAParams : "") + (s_dbHAEnabled ? "&loadBalanceStrategy=" + loadBalanceStrategy : ""), cloudUsername, cloudPassword); final KeyedObjectPoolFactory poolableObjFactory = (cloudPoolPreparedStatements ? new StackKeyedObjectPoolFactory() : null); @@ -1116,7 +1116,7 @@ public class TransactionLegacy { usageMaxWait, usageMaxIdle); final ConnectionFactory usageConnectionFactory = new DriverManagerConnectionFactory("jdbc:mysql://" + usageHost + (s_dbHAEnabled ? "," + dbProps.getProperty("db.cloud.slaves") : "") + ":" + usagePort + "/" + usageDbName + - "?autoReconnect=" + usageAutoReconnect + (usageUrl != null ? "&" + usageUrl : "") + (s_dbHAEnabled ? "&" + getDBHAParams("usage", dbProps) : ""), usageUsername, usagePassword); + "?autoReconnect=" + usageAutoReconnect + (usageUrl != null ? "&" + usageUrl : "") + (s_dbHAEnabled ? "&" + getDBHAParams("usage", dbProps) : "") + (s_dbHAEnabled ? "&loadBalanceStrategy=" + loadBalanceStrategy : ""), usageUsername, usagePassword); final PoolableConnectionFactory usagePoolableConnectionFactory = new PoolableConnectionFactory(usageConnectionFactory, usageConnectionPool, new StackKeyedObjectPoolFactory(), null, false, false); @@ -1129,7 +1129,7 @@ public class TransactionLegacy { final GenericObjectPool awsapiConnectionPool = new GenericObjectPool(null, usageMaxActive, GenericObjectPool.DEFAULT_WHEN_EXHAUSTED_ACTION, usageMaxWait, usageMaxIdle); final ConnectionFactory awsapiConnectionFactory = new DriverManagerConnectionFactory("jdbc:mysql://" + cloudHost + (s_dbHAEnabled ? "," + cloudSlaves : "") + ":" + cloudPort + "/" + awsapiDbName + - "?autoReconnect=" + cloudAutoReconnect + (s_dbHAEnabled ? "&" + cloudDbHAParams : ""), cloudUsername, cloudPassword); + "?autoReconnect=" + cloudAutoReconnect + (s_dbHAEnabled ? "&" + cloudDbHAParams : "") + (s_dbHAEnabled ? "&loadBalanceStrategy=" + loadBalanceStrategy : ""), cloudUsername, cloudPassword); final PoolableConnectionFactory awsapiPoolableConnectionFactory = new PoolableConnectionFactory(awsapiConnectionFactory, awsapiConnectionPool, new StackKeyedObjectPoolFactory(), null, false, false);