From b79f949e1bd9d62b3ebcce424608cb4b22e69897 Mon Sep 17 00:00:00 2001 From: Anthony Xu Date: Tue, 21 Jan 2014 17:55:09 -0800 Subject: [PATCH] CLOUDSTACK-5923: CS doesn't do master switch for XS any more, CS will depend on XS HA to do master switch, XS HA needs to be enabled. Conflicts: plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/discoverer/XcpServerDiscoverer.java plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/XenServerConnectionPool.java --- .../xen/discoverer/XcpServerDiscoverer.java | 11 +- .../xen/resource/CitrixResourceBase.java | 53 +- .../xen/resource/XenServerConnectionPool.java | 456 ++---------------- 3 files changed, 55 insertions(+), 465 deletions(-) diff --git a/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/discoverer/XcpServerDiscoverer.java b/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/discoverer/XcpServerDiscoverer.java index cd1b30b9f71..58fe0159815 100755 --- a/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/discoverer/XcpServerDiscoverer.java +++ b/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/discoverer/XcpServerDiscoverer.java @@ -195,8 +195,7 @@ public class XcpServerDiscoverer extends DiscovererBase implements Discoverer, L String hostIp = ia.getHostAddress(); Queue pass = new LinkedList(); pass.add(password); - String masterIp = _connPool.getMasterIp(hostIp, username, pass); - conn = _connPool.masterConnect(masterIp, username, pass); + conn = _connPool.getConnect(hostIp, username, pass); if (conn == null) { String msg = "Unable to get a connection to " + url; s_logger.debug(msg); @@ -398,7 +397,7 @@ public class XcpServerDiscoverer extends DiscovererBase implements Discoverer, L password = host.getDetail("password"); pass.add(password); String address = host.getPrivateIpAddress(); - Connection hostConn = _connPool.slaveConnect(address, username, pass); + Connection hostConn = _connPool.getConnect(address, username, pass); if (hostConn == null) { continue; } @@ -411,9 +410,9 @@ public class XcpServerDiscoverer extends DiscovererBase implements Discoverer, L } catch (Exception e) { s_logger.warn("Can not get master ip address from host " + address); } finally { - try { - Session.localLogout(hostConn); - } catch (Exception e) { + try{ + Session.logout(hostConn); + } catch (Exception e ) { } hostConn.dispose(); hostConn = null; diff --git a/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java b/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java index 5aed2141872..9abfee345e7 100644 --- a/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java +++ b/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java @@ -424,23 +424,12 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe } protected boolean pingXenServer() { - Session slaveSession = null; - Connection slaveConn = null; + Connection conn = getConnection(); try { - URL slaveUrl = null; - slaveUrl = ConnPool.getURL(_host.ip); - slaveConn = new Connection(slaveUrl, 10); - slaveSession = ConnPool.slaveLocalLoginWithPassword(slaveConn, _username, _password); + callHostPlugin(conn, "echo", "main"); return true; } catch (Exception e) { - } finally { - if (slaveSession != null) { - try { - Session.localLogout(slaveConn); - } catch (Exception e) { - } - slaveConn.dispose(); - } + s_logger.debug("cannot ping host " + _host.ip + " due to " + e.toString(), e); } return false; } @@ -6020,9 +6009,9 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe } private void CheckXenHostInfo() throws ConfigurationException { - Connection conn = ConnPool.slaveConnect(_host.ip, _username, _password); - if (conn == null) { - throw new ConfigurationException("Can not create slave connection to " + _host.ip); + Connection conn = _connPool.getConnect(_host.ip, _username, _password); + if( conn == null ) { + throw new ConfigurationException("Can not create connection to " + _host.ip); } try { Host.Record hostRec = null; @@ -6042,7 +6031,7 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe } } finally { try { - Session.localLogout(conn); + Session.logout(conn); } catch (Exception e) { } } @@ -7311,35 +7300,11 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe Host.Record hostr = poolr.master.getRecord(conn); if (_host.uuid.equals(hostr.uuid)) { - boolean mastermigrated = false; Map hostMap = Host.getAllRecords(conn); - if (hostMap.size() != 1) { - Host newMaster = null; - Host.Record newMasterRecord = null; - for (Map.Entry entry : hostMap.entrySet()) { - if (_host.uuid.equals(entry.getValue().uuid)) { - continue; - } - newMaster = entry.getKey(); - newMasterRecord = entry.getValue(); - s_logger.debug("New master for the XenPool is " + newMasterRecord.uuid + " : " + newMasterRecord.address); - try { - ConnPool.switchMaster(_host.ip, _host.pool, conn, newMaster, _username, _password, _wait); - mastermigrated = true; - break; - } catch (Exception e) { - s_logger.warn("Unable to switch the new master to " + newMasterRecord.uuid + ": " + newMasterRecord.address + " due to " + e.toString()); - } - } - } else { - s_logger.debug("This is last host to eject, so don't need to eject: " + hostuuid); - return new Answer(cmd); - } - if (!mastermigrated) { - String msg = "this host is master, and cannot designate a new master"; + if (hostMap.size() > 1) { + String msg = "This host is XS master, please designate a new XS master throught XenCenter before you delete this host from CS"; s_logger.debug(msg); return new Answer(cmd, false, msg); - } } diff --git a/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/XenServerConnectionPool.java b/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/XenServerConnectionPool.java index e797449b0cb..572d08e2f5b 100644 --- a/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/XenServerConnectionPool.java +++ b/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/XenServerConnectionPool.java @@ -26,7 +26,6 @@ import java.util.HashMap; import java.util.Map; import java.util.Properties; import java.util.Queue; -import java.util.Set; import javax.net.ssl.HostnameVerifier; import javax.net.ssl.HttpsURLConnection; @@ -43,7 +42,6 @@ import com.xensource.xenapi.Pool; import com.xensource.xenapi.Session; import com.xensource.xenapi.Types; import com.xensource.xenapi.Types.BadServerResponse; -import com.xensource.xenapi.Types.UuidInvalid; import com.xensource.xenapi.Types.XenAPIException; import com.cloud.utils.NumbersUtil; @@ -55,7 +53,6 @@ public class XenServerConnectionPool { protected HashMap _conns = new HashMap(); protected int _retries; protected int _interval; - protected static boolean s_managePool = true; protected static long s_sleepOnError = 10 * 1000; // in ms static { File file = PropertiesUtil.findConfigFile("environment.properties"); @@ -68,15 +65,11 @@ public class XenServerConnectionPool { final Properties props = new Properties(); props.load(finputstream); finputstream.close(); - String search = props.getProperty("manage.xenserver.pool.master"); - if (search != null) { - s_managePool = Boolean.parseBoolean(search); - } - search = props.getProperty("sleep.interval.on.error"); + String search = props.getProperty("sleep.interval.on.error"); if (search != null) { s_sleepOnError = NumbersUtil.parseInterval(search, 10) * 1000; } - s_logger.info("XenServer Connection Pool Configs: manage.xenserver.pool.master=" + s_managePool + "; sleep.interval.on.error=" + s_sleepOnError); + s_logger.info("XenServer Connection Pool Configs: sleep.interval.on.error=" + s_sleepOnError); } catch (FileNotFoundException e) { s_logger.debug("File is not found", e); } catch (IOException e) { @@ -214,288 +207,27 @@ public class XenServerConnectionPool { return false; } - public void switchMaster(String slaveIp, String poolUuid, Connection conn, Host host, String username, Queue password, int wait) throws XmlRpcException, - XenAPIException { - synchronized (poolUuid.intern()) { - String masterIp = host.getAddress(conn); - s_logger.debug("Designating the new master to " + masterIp); - Pool.designateNewMaster(conn, host); - Connection slaveConn = null; - Connection masterConn = null; - int retry = 30; - for (int i = 0; i < retry; i++) { - forceSleep(5); - try { - if (s_logger.isDebugEnabled()) { - s_logger.debug("Logging on as the slave to " + slaveIp); - } - slaveConn = null; - masterConn = null; - Session slaveSession = null; - - slaveConn = new Connection(getURL(slaveIp), 10); - slaveSession = slaveLocalLoginWithPassword(slaveConn, username, password); - - if (s_logger.isDebugEnabled()) { - s_logger.debug("Slave logon successful. session= " + slaveSession); - } - - Pool.Record pr = getPoolRecord(slaveConn); - Host master = pr.master; - String ma = master.getAddress(slaveConn); - if (!ma.trim().equals(masterIp.trim())) { - continue; - } - s_logger.debug("Logging on as the master to " + masterIp); - masterConn = new Connection(getURL(masterIp), 10); - loginWithPassword(masterConn, username, password, APIVersion.latest().toString()); - removeConnect(poolUuid); - ensurePoolIntegrity(masterConn, masterIp, username, password, wait); - return; - } catch (Types.HostIsSlave e) { - s_logger.debug("HostIsSlaveException: Still waiting for the conversion to the master"); - } catch (XmlRpcException e) { - s_logger.debug("XmlRpcException: Still waiting for the conversion to the master " + e.getMessage()); - } catch (Exception e) { - s_logger.debug("Exception: Still waiting for the conversion to the master" + e.getMessage()); - } finally { - if (masterConn != null) { - try { - Session.logout(masterConn); - } catch (Exception e) { - s_logger.debug("Unable to log out of session: " + e.getMessage()); - } - masterConn.dispose(); - masterConn = null; - } - localLogout(slaveConn); - slaveConn = null; - } - } - throw new CloudRuntimeException("Unable to logon to the new master after " + retry + " retries"); - } - } - - private void localLogout(Connection conn) { - if (conn == null) - return; + + public Connection getConnect(String ip, String username, Queue password) { + Connection conn = new Connection(getURL(ip), 10); try { - if (s_logger.isTraceEnabled()) { - s_logger.trace("Logging out of the session " + conn.getSessionReference()); - } - Session.localLogout(conn); - } catch (Exception e) { - s_logger.debug("localLogout has problem " + e.getMessage()); - } finally { - conn.dispose(); - conn = null; - } - } - - public Connection slaveConnect(String ip, String username, Queue password) { - Connection conn = null; - try { - conn = new Connection(getURL(ip), 10); - slaveLocalLoginWithPassword(conn, username, password); - return conn; - } catch (Exception e) { - s_logger.debug("Failed to slave local login to " + ip); - } - return null; - } - - public Connection masterConnect(String ip, String username, Queue password) { - Connection conn = null; - try { - conn = new Connection(getURL(ip), 10); - s_logger.debug("Logging on as the master to " + ip); loginWithPassword(conn, username, password, APIVersion.latest().toString()); - return conn; - } catch (Exception e) { - s_logger.debug("Failed to slave local login to " + ip); - } - throw new RuntimeException("can not log in to master " + ip); - } - - public String getMasterIp(String ip, String username, Queue password) throws XenAPIException { - Connection slaveConn = null; - try { - slaveConn = new Connection(getURL(ip), 10); - slaveLocalLoginWithPassword(slaveConn, username, password); - - if (s_logger.isDebugEnabled()) { - s_logger.debug("Slave logon to " + ip); - } - String masterIp = null; - Pool.Record pr = getPoolRecord(slaveConn); - Host master = pr.master; - masterIp = master.getAddress(slaveConn); - return masterIp; - } catch (Types.SessionAuthenticationFailed e) { - s_logger.debug("Failed to slave local login to " + ip + " due to " + e.toString()); - throw e; - } catch (Exception e) { - s_logger.debug("Failed to slave local login to " + ip + " due to " + e.toString()); - } finally { - localLogout(slaveConn); - slaveConn = null; - } - throw new RuntimeException("can not get master ip"); - } - - void PoolEmergencyTransitionToMaster(String slaveIp, String username, Queue password) { - if (!s_managePool) { - if (s_logger.isDebugEnabled()) { - s_logger.debug("Don't manage pool on error so sleeping for " + s_sleepOnError); - try { - Thread.sleep(s_sleepOnError); - } catch (InterruptedException ie) { - } - } - return; - } - - Connection slaveConn = null; - Connection c = null; - try { - s_logger.debug("Trying to transition master to " + slaveIp); - slaveConn = new Connection(getURL(slaveIp), 10); - slaveLocalLoginWithPassword(slaveConn, username, password); - Pool.emergencyTransitionToMaster(slaveConn); - // restart xapi in 10 sec - forceSleep(10); - // check if the master of this host is set correctly. - c = new Connection(getURL(slaveIp), 10); - for (int i = 0; i < 30; i++) { - try { - loginWithPassword(c, username, password, APIVersion.latest().toString()); - s_logger.debug("Succeeded to transition master to " + slaveIp); - return; - } catch (Types.HostIsSlave e) { - s_logger.debug("HostIsSlave: Still waiting for the conversion to the master " + slaveIp); - } catch (Exception e) { - s_logger.debug("Exception: Still waiting for the conversion to the master"); - } - forceSleep(2); - } - throw new RuntimeException("EmergencyTransitionToMaster failed after retry 30 times"); - } catch (Exception e) { - throw new RuntimeException("EmergencyTransitionToMaster failed due to " + e.getMessage()); - } finally { - localLogout(slaveConn); - slaveConn = null; - if (c != null) { - try { - Session.logout(c); - c.dispose(); - } catch (Exception e) { - } - } - } - - } - - private void PoolEmergencyResetMaster(String slaveIp, String masterIp, String username, Queue password) { - if (!s_managePool) { - if (s_logger.isDebugEnabled()) { - s_logger.debug("Don't manage pool on error so sleeping for " + s_sleepOnError); - try { - Thread.sleep(s_sleepOnError); - } catch (InterruptedException ie) { - } - } - return; - } - - Connection slaveConn = null; - try { - s_logger.debug("Trying to reset master of slave " + slaveIp + " to " + masterIp); - slaveConn = new Connection(getURL(slaveIp), 10); - slaveLocalLoginWithPassword(slaveConn, username, password); - Pool.emergencyResetMaster(slaveConn, masterIp); - forceSleep(10); - for (int i = 0; i < 30; i++) { - try { - slaveLocalLoginWithPassword(slaveConn, username, password); - Pool.Record pr = getPoolRecord(slaveConn); - String mIp = pr.master.getAddress(slaveConn); - if (mIp.trim().equals(masterIp.trim())) { - s_logger.debug("Succeeded to reset master of slave " + slaveIp + " to " + masterIp); - return; - } - } catch (Exception e) { - } finally { - localLogout(slaveConn); - slaveConn = null; - } - // wait 2 second - forceSleep(2); - } - throw new CloudRuntimeException("Unable to reset master of slave " + slaveIp + " to " + masterIp + "after 30 retry"); - } catch (Exception e) { - throw new CloudRuntimeException("Unable to reset master of slave " + slaveIp + " to " + masterIp + " due to " + e.toString()); - } finally { - localLogout(slaveConn); - slaveConn = null; - } - } - - protected void ensurePoolIntegrity(Connection conn, String masterIp, String username, Queue password, int wait) { - try { - // try recoverSlave first - Set rcSlaves = Pool.recoverSlaves(conn); - // wait 10 second - forceSleep(10); - for (Host slave : rcSlaves) { - for (int i = 0; i < 30; i++) { - Connection slaveConn = null; - try { - - String slaveIp = slave.getAddress(conn); - s_logger.debug("Logging on as the slave to " + slaveIp); - slaveConn = new Connection(getURL(slaveIp), 10); - slaveLocalLoginWithPassword(slaveConn, username, password); - Pool.Record pr = getPoolRecord(slaveConn); - String mIp = pr.master.getAddress(slaveConn); - if (mIp.trim().equals(masterIp.trim())) { - break; - } - } catch (Exception e) { - } finally { - localLogout(slaveConn); - slaveConn = null; - } - // wait 2 second - forceSleep(2); - } - } - // then try emergency reset master - Set slaves = Host.getAll(conn); - for (Host slave : slaves) { - String slaveIp = slave.getAddress(conn); - Connection slaveConn = null; - try { - s_logger.debug("Logging on as the slave to " + slaveIp); - - slaveConn = new Connection(getURL(slaveIp), 10); - slaveLocalLoginWithPassword(slaveConn, username, password); - Pool.Record slavePoolr = getPoolRecord(slaveConn); - String ip = slavePoolr.master.getAddress(slaveConn); - if (!masterIp.trim().equals(ip.trim())) { - PoolEmergencyResetMaster(slaveIp, masterIp, username, password); - } - } catch (Exception e) { - s_logger.debug("Unable to login to slave " + slaveIp + " error " + e.getMessage()); - } finally { - localLogout(slaveConn); - slaveConn = null; - } + } catch (Types.HostIsSlave e) { + String maddress = e.masterIPAddress; + conn = new Connection(getURL(maddress), 10); + try { + loginWithPassword(conn, username, password, APIVersion.latest().toString()); + } catch (Exception e1) { + String msg = "Unable to create master connection to host(" + maddress +") , due to " + e1.toString(); + s_logger.debug(msg); + throw new CloudRuntimeException(msg, e1); } } catch (Exception e) { - if (s_logger.isDebugEnabled()) { - s_logger.debug("Catch " + e.getClass().getName() + " due to " + e.toString()); - } + String msg = "Unable to create master connection to host(" + ip +") , due to " + e.toString(); + s_logger.debug(msg); + throw new CloudRuntimeException(msg, e); } + return conn; } public URL getURL(String ip) { @@ -512,156 +244,50 @@ public class XenServerConnectionPool { public Connection connect(String hostUuid, String poolUuid, String ipAddress, String username, Queue password, int wait) { XenServerConnection mConn = null; - Connection sConn = null; - String masterIp = null; if (hostUuid == null || poolUuid == null || ipAddress == null || username == null || password == null) { String msg = "Connect some parameter are null hostUuid:" + hostUuid + " ,poolUuid:" + poolUuid + " ,ipAddress:" + ipAddress; s_logger.debug(msg); throw new CloudRuntimeException(msg); } - Host host = null; synchronized (poolUuid.intern()) { // Let's see if it is an existing connection. mConn = getConnect(poolUuid); - if (mConn != null) { - try { - host = Host.getByUuid(mConn, hostUuid); - } catch (Types.SessionInvalid e) { - s_logger.debug("Session thgrough ip " + mConn.getIp() + " is invalid for pool(" + poolUuid + ") due to " + e.toString()); - try { - loginWithPassword(mConn, mConn.getUsername(), mConn.getPassword(), APIVersion.latest().toString()); - } catch (Exception e1) { - if (s_logger.isDebugEnabled()) { - s_logger.debug("connect through IP(" + mConn.getIp() + " for pool(" + poolUuid + ") is broken due to " + e.toString()); - } - removeConnect(poolUuid); - mConn = null; - } - } catch (UuidInvalid e) { - String msg = - "Host(" + hostUuid + ") doesn't belong to pool(" + poolUuid + "), please execute 'xe pool-join master-address=" + mConn.getIp() + - " master-username=" + mConn.getUsername(); - if (s_logger.isDebugEnabled()) { - s_logger.debug(msg); - } - throw new CloudRuntimeException(msg, e); - } catch (Exception e) { + if (mConn != null){ + try{ + Host.getByUuid(mConn, hostUuid); + } catch (Exception e) { if (s_logger.isDebugEnabled()) { s_logger.debug("connect through IP(" + mConn.getIp() + " for pool(" + poolUuid + ") is broken due to " + e.toString()); } removeConnect(poolUuid); mConn = null; } - } - - if (mConn == null) { + } + + if ( mConn == null ) { + mConn = new XenServerConnection(getURL(ipAddress), ipAddress, username, password, _retries, _interval, wait); try { + loginWithPassword(mConn, username, password, APIVersion.latest().toString()); + } catch (Types.HostIsSlave e) { + String maddress = e.masterIPAddress; + mConn = new XenServerConnection(getURL(maddress), maddress, username, password, _retries, _interval, wait); try { - if (s_logger.isDebugEnabled()) { - s_logger.debug("Logging on as the slave to " + ipAddress); - } - sConn = new Connection(getURL(ipAddress), 5); - slaveLocalLoginWithPassword(sConn, username, password); - } catch (Exception e) { - String msg = "Unable to create slave connection to host(" + hostUuid + ") due to " + e.toString(); - if (s_logger.isDebugEnabled()) { - s_logger.debug(msg); - } - throw new CloudRuntimeException(msg, e); - } - Pool.Record pr = null; - try { - pr = getPoolRecord(sConn); - } catch (Exception e) { - PoolEmergencyTransitionToMaster(ipAddress, username, password); - mConn = new XenServerConnection(getURL(ipAddress), ipAddress, username, password, _retries, _interval, wait); - try { - loginWithPassword(mConn, username, password, APIVersion.latest().toString()); - pr = getPoolRecord(mConn); - } catch (Exception e1) { - String msg = "Unable to create master connection to host(" + hostUuid + ") after transition it to master, due to " + e1.toString(); - if (s_logger.isDebugEnabled()) { - s_logger.debug(msg); - } - throw new CloudRuntimeException(msg, e1); - } - if (!pr.uuid.equals(poolUuid)) { - String msg = "host(" + hostUuid + ") should be in pool(" + poolUuid + "), but it is actually in pool(" + pr.uuid + ")"; - if (s_logger.isDebugEnabled()) { - s_logger.debug(msg); - } - throw new CloudRuntimeException(msg); - } else { - if (s_managePool) { - ensurePoolIntegrity(mConn, ipAddress, username, password, wait); - } - addConnect(poolUuid, mConn); - return mConn; - } - } - if (!pr.uuid.equals(poolUuid)) { - String msg = "host(" + hostUuid + ") should be in pool(" + poolUuid + "), but it is actually in pool(" + pr.uuid + ")"; - if (s_logger.isDebugEnabled()) { - s_logger.debug(msg); - } - throw new CloudRuntimeException(msg); - } - try { - masterIp = pr.master.getAddress(sConn); - mConn = new XenServerConnection(getURL(masterIp), masterIp, username, password, _retries, _interval, wait); loginWithPassword(mConn, username, password, APIVersion.latest().toString()); - addConnect(poolUuid, mConn); - return mConn; - } catch (Exception e) { - String msg = "Unable to logon in " + masterIp + " as master in pool(" + poolUuid + ")"; - if (s_logger.isDebugEnabled()) { - s_logger.debug(msg); - } - throw new CloudRuntimeException(msg); - } - } finally { - localLogout(sConn); - sConn = null; + } catch (Exception e1) { + String msg = "Unable to create master connection to host(" + maddress +") , due to " + e1.toString(); + s_logger.debug(msg); + throw new CloudRuntimeException(msg, e1); + + } + } catch (Exception e) { + String msg = "Unable to create master connection to host(" + ipAddress +") , due to " + e.toString(); + s_logger.debug(msg); + throw new CloudRuntimeException(msg, e); } + addConnect(poolUuid, mConn); } } - if (mConn != null) { - if (s_managePool) { - try { - Map args = new HashMap(); - host.callPlugin(mConn, "echo", "main", args); - } catch (Types.SessionInvalid e) { - if (s_logger.isDebugEnabled()) { - String msg = "Catch Exception: " + e.getClass().getName() + " Can't connect host " + ipAddress + " due to " + e.toString(); - s_logger.debug(msg); - } - PoolEmergencyResetMaster(ipAddress, mConn.getIp(), mConn.getUsername(), mConn.getPassword()); - } catch (Types.CannotContactHost e) { - if (s_logger.isDebugEnabled()) { - String msg = "Catch Exception: " + e.getClass().getName() + " Can't connect host " + ipAddress + " due to " + e.toString(); - s_logger.debug(msg); - } - PoolEmergencyResetMaster(ipAddress, mConn.getIp(), mConn.getUsername(), mConn.getPassword()); - } catch (Types.HostOffline e) { - if (s_logger.isDebugEnabled()) { - String msg = "Catch Exception: " + e.getClass().getName() + " Host is offline " + ipAddress + " due to " + e.toString(); - s_logger.debug(msg); - } - PoolEmergencyResetMaster(ipAddress, mConn.getIp(), mConn.getUsername(), mConn.getPassword()); - } catch (Types.HostNotLive e) { - String msg = "Catch Exception: " + e.getClass().getName() + " Host Not Live " + ipAddress + " due to " + e.toString(); - if (s_logger.isDebugEnabled()) { - s_logger.debug(msg); - } - PoolEmergencyResetMaster(ipAddress, mConn.getIp(), mConn.getUsername(), mConn.getPassword()); - } catch (Exception e) { - String msg = "Echo test failed on host " + hostUuid + " IP " + ipAddress; - s_logger.warn(msg, e); - throw new CloudRuntimeException(msg, e); - } - } - } return mConn; }