From f6258dae08b18489d86561ea2bcbda21304cf95b Mon Sep 17 00:00:00 2001 From: Alex Huang Date: Mon, 25 Apr 2011 09:29:38 -0700 Subject: [PATCH] bug 9445: Signal alert for the host if a primary storage pool was unavailable on that host --- .../cloud/exception/ConnectionException.java | 17 +++++- .../cloud/agent/manager/AgentManagerImpl.java | 23 ++++--- .../src/com/cloud/storage/StorageManager.java | 53 ++++++++-------- .../com/cloud/storage/StorageManagerImpl.java | 61 +++++++++---------- .../storage/listener/StoragePoolMonitor.java | 8 ++- 5 files changed, 94 insertions(+), 68 deletions(-) diff --git a/api/src/com/cloud/exception/ConnectionException.java b/api/src/com/cloud/exception/ConnectionException.java index 02386a274b5..0a467bd736f 100644 --- a/api/src/com/cloud/exception/ConnectionException.java +++ b/api/src/com/cloud/exception/ConnectionException.java @@ -19,11 +19,26 @@ package com.cloud.exception; import com.cloud.utils.SerialVersionUID; +/** + * ConnectionException is thrown by Listeners while processing the startup + * command. There are two uses for this exception and they are distinguished + * by the boolean flag. + * 1. If the flag is set to true, there is an unexpected error during the + * processing. Upon receiving this exception, the AgentManager will + * immediately place the agent under alert. When the function to enable + * to disable the agent, the agent is disabled. + * 2. If the flag is set to false, the listener has decided that the resource + * should be disconnected and reconnected to "refresh" all resource + * information. This is useful when the Listener needed to perform setup + * on the agent and decided it is best to flush connection and reconnect. + * It is important that the Listener does not fall into a loop in this + * situation where it keeps throwing ConnectionException. + */ public class ConnectionException extends Exception { private static final long serialVersionUID = SerialVersionUID.ConnectionException; boolean _error; - + public ConnectionException(boolean setupError, String msg) { this(setupError, msg, null); } diff --git a/server/src/com/cloud/agent/manager/AgentManagerImpl.java b/server/src/com/cloud/agent/manager/AgentManagerImpl.java index d54b43aefee..5d5f7694601 100755 --- a/server/src/com/cloud/agent/manager/AgentManagerImpl.java +++ b/server/src/com/cloud/agent/manager/AgentManagerImpl.java @@ -1554,15 +1554,22 @@ public class AgentManagerImpl implements AgentManager, HandlerFactory, ResourceS for (int i = 0; i < cmd.length; i++) { try { monitor.second().processConnect(host, cmd[i]); - } catch (ConnectionException e) { - if (e.isSetupError()) { - s_logger.warn("Monitor " + monitor.second().getClass().getSimpleName() + " says there is an error in the connect process for " + hostId + " due to " + e.getMessage()); - handleDisconnect(attache, Event.AgentDisconnected, false); - throw e; + } catch (Exception e) { + if (e instanceof ConnectionException) { + ConnectionException ce = (ConnectionException)e; + if (ce.isSetupError()) { + s_logger.warn("Monitor " + monitor.second().getClass().getSimpleName() + " says there is an error in the connect process for " + hostId + " due to " + e.getMessage()); + handleDisconnect(attache, Event.AgentDisconnected, false); + throw ce; + } else { + s_logger.info("Monitor " + monitor.second().getClass().getSimpleName() + " says not to continue the connect process for " + hostId + " due to " + e.getMessage()); + handleDisconnect(attache, Event.ShutdownRequested, false); + return attache; + } } else { - s_logger.info("Monitor " + monitor.second().getClass().getSimpleName() + " says not to continue the connect process for " + hostId + " due to " + e.getMessage()); - handleDisconnect(attache, Event.ShutdownRequested, false); - return attache; + s_logger.error("Monitor " + monitor.second().getClass().getSimpleName() + " says there is an error in the connect process for " + hostId + " due to " + e.getMessage(), e); + handleDisconnect(attache, Event.AgentDisconnected, false); + throw new CloudRuntimeException("Unable to connect " + attache.getId(), e); } } } diff --git a/server/src/com/cloud/storage/StorageManager.java b/server/src/com/cloud/storage/StorageManager.java index 82c1c8acbfd..e1802fd24af 100755 --- a/server/src/com/cloud/storage/StorageManager.java +++ b/server/src/com/cloud/storage/StorageManager.java @@ -17,30 +17,30 @@ */ package com.cloud.storage; -import java.util.List; - -import com.cloud.agent.api.Answer; -import com.cloud.agent.api.Command; -import com.cloud.agent.manager.Commands; -import com.cloud.capacity.CapacityVO; -import com.cloud.dc.DataCenterVO; -import com.cloud.dc.HostPodVO; -import com.cloud.deploy.DeployDestination; -import com.cloud.exception.ConcurrentOperationException; -import com.cloud.exception.InsufficientStorageCapacityException; -import com.cloud.exception.StorageUnavailableException; -import com.cloud.host.Host; -import com.cloud.host.HostVO; -import com.cloud.hypervisor.Hypervisor.HypervisorType; -import com.cloud.service.ServiceOfferingVO; -import com.cloud.storage.Volume.Type; -import com.cloud.user.Account; -import com.cloud.utils.Pair; -import com.cloud.utils.component.Manager; -import com.cloud.vm.DiskProfile; -import com.cloud.vm.VMInstanceVO; -import com.cloud.vm.VirtualMachine; -import com.cloud.vm.VirtualMachineProfile; +import java.util.List; + +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.Command; +import com.cloud.agent.manager.Commands; +import com.cloud.capacity.CapacityVO; +import com.cloud.dc.DataCenterVO; +import com.cloud.dc.HostPodVO; +import com.cloud.deploy.DeployDestination; +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientStorageCapacityException; +import com.cloud.exception.StorageUnavailableException; +import com.cloud.host.Host; +import com.cloud.host.HostVO; +import com.cloud.hypervisor.Hypervisor.HypervisorType; +import com.cloud.service.ServiceOfferingVO; +import com.cloud.storage.Volume.Type; +import com.cloud.user.Account; +import com.cloud.utils.Pair; +import com.cloud.utils.component.Manager; +import com.cloud.vm.DiskProfile; +import com.cloud.vm.VMInstanceVO; +import com.cloud.vm.VirtualMachine; +import com.cloud.vm.VirtualMachineProfile; public interface StorageManager extends Manager { boolean canVmRestartOnAnotherServer(long vmId); @@ -76,9 +76,10 @@ public interface StorageManager extends Manager { /** * Add a pool to a host * @param hostId - * @param pool + * @param pool + * @throws StorageUnavailableException */ - boolean addPoolToHost(long hostId, StoragePoolVO pool); + void addPoolToHost(long hostId, StoragePoolVO pool) throws StorageUnavailableException; /** * Moves a volume from its current storage pool to a storage pool with enough capacity in the specified zone, pod, or cluster diff --git a/server/src/com/cloud/storage/StorageManagerImpl.java b/server/src/com/cloud/storage/StorageManagerImpl.java index c3d902b382a..43359d3c6aa 100755 --- a/server/src/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/com/cloud/storage/StorageManagerImpl.java @@ -1171,9 +1171,11 @@ public class StorageManagerImpl implements StorageManager, StorageService, Manag s_logger.debug("In createPool Adding the pool to each of the hosts"); List poolHosts = new ArrayList(); for (HostVO h : allHosts) { - success = addPoolToHost(h.getId(), pool); - if (success) { + try { + addPoolToHost(h.getId(), pool); poolHosts.add(h); + } catch (Exception e) { + s_logger.warn("Unable to establish a connection between " + h + " and " + pool, e); } } @@ -1356,45 +1358,42 @@ public class StorageManagerImpl implements StorageManager, StorageService, Manag } @Override - public boolean addPoolToHost(long hostId, StoragePoolVO pool) { + public void addPoolToHost(long hostId, StoragePoolVO pool) throws StorageUnavailableException { s_logger.debug("Adding pool " + pool.getName() + " to host " + hostId); if (pool.getPoolType() != StoragePoolType.NetworkFilesystem && pool.getPoolType() != StoragePoolType.Filesystem && pool.getPoolType() != StoragePoolType.IscsiLUN && pool.getPoolType() != StoragePoolType.Iscsi && pool.getPoolType() != StoragePoolType.VMFS && pool.getPoolType() != StoragePoolType.SharedMountPoint && pool.getPoolType() != StoragePoolType.PreSetup) { - s_logger.warn(" Doesn't support storage pool type " + pool.getPoolType()); - return false; + throw new CloudRuntimeException("Doesn't support storage pool type " + pool.getPoolType()); } ModifyStoragePoolCommand cmd = new ModifyStoragePoolCommand(true, pool); final Answer answer = _agentMgr.easySend(hostId, cmd); - if (answer != null) { - if (answer.getResult() == false) { - String msg = "Add host failed due to ModifyStoragePoolCommand failed" + answer.getDetails(); - _alertMgr.sendAlert(AlertManager.ALERT_TYPE_HOST, pool.getDataCenterId(), pool.getPodId(), msg, msg); - s_logger.warn(msg); - return false; - } - if (answer instanceof ModifyStoragePoolAnswer) { - ModifyStoragePoolAnswer mspAnswer = (ModifyStoragePoolAnswer) answer; - - StoragePoolHostVO poolHost = _poolHostDao.findByPoolHost(pool.getId(), hostId); - if (poolHost == null) { - poolHost = new StoragePoolHostVO(pool.getId(), hostId, mspAnswer.getPoolInfo().getLocalPath().replaceAll("//", "/")); - _poolHostDao.persist(poolHost); - } else { - poolHost.setLocalPath(mspAnswer.getPoolInfo().getLocalPath().replaceAll("//", "/")); - } - pool.setAvailableBytes(mspAnswer.getPoolInfo().getAvailableBytes()); - pool.setCapacityBytes(mspAnswer.getPoolInfo().getCapacityBytes()); - _storagePoolDao.update(pool.getId(), pool); - return true; - } - - } else { - return false; + if (answer == null) { + throw new StorageUnavailableException("Unable to get an answer to the modify storage pool command", pool.getId()); } - return false; + + if (!answer.getResult()) { + String msg = "Add host failed due to ModifyStoragePoolCommand failed" + answer.getDetails(); + _alertMgr.sendAlert(AlertManager.ALERT_TYPE_HOST, pool.getDataCenterId(), pool.getPodId(), msg, msg); + throw new StorageUnavailableException("Unable establish connection from storage head to storage pool " + pool.getId() + " due to " + answer.getDetails(), pool.getId()); + } + + assert (answer instanceof ModifyStoragePoolAnswer) : "Well, now why won't you actually return the ModifyStoragePoolAnswer when it's ModifyStoragePoolCommand? Pool=" + pool.getId() + "Host=" + hostId; + ModifyStoragePoolAnswer mspAnswer = (ModifyStoragePoolAnswer) answer; + + StoragePoolHostVO poolHost = _poolHostDao.findByPoolHost(pool.getId(), hostId); + if (poolHost == null) { + poolHost = new StoragePoolHostVO(pool.getId(), hostId, mspAnswer.getPoolInfo().getLocalPath().replaceAll("//", "/")); + _poolHostDao.persist(poolHost); + } else { + poolHost.setLocalPath(mspAnswer.getPoolInfo().getLocalPath().replaceAll("//", "/")); + } + pool.setAvailableBytes(mspAnswer.getPoolInfo().getAvailableBytes()); + pool.setCapacityBytes(mspAnswer.getPoolInfo().getCapacityBytes()); + _storagePoolDao.update(pool.getId(), pool); + + s_logger.info("Connection established between " + pool + " host + " + hostId); } @Override diff --git a/server/src/com/cloud/storage/listener/StoragePoolMonitor.java b/server/src/com/cloud/storage/listener/StoragePoolMonitor.java index aac1ca6e613..c5cb23bc3c6 100755 --- a/server/src/com/cloud/storage/listener/StoragePoolMonitor.java +++ b/server/src/com/cloud/storage/listener/StoragePoolMonitor.java @@ -28,6 +28,7 @@ import com.cloud.agent.api.Answer; import com.cloud.agent.api.Command; import com.cloud.agent.api.StartupCommand; import com.cloud.agent.api.StartupRoutingCommand; +import com.cloud.exception.ConnectionException; import com.cloud.host.HostVO; import com.cloud.host.Status; import com.cloud.host.dao.HostDao; @@ -66,7 +67,7 @@ public class StoragePoolMonitor implements Listener { } @Override - public void processConnect(HostVO host, StartupCommand cmd) { + public void processConnect(HostVO host, StartupCommand cmd) throws ConnectionException { if (cmd instanceof StartupRoutingCommand) { StartupRoutingCommand scCmd = (StartupRoutingCommand)cmd; if (scCmd.getHypervisorType() == HypervisorType.XenServer || scCmd.getHypervisorType() == HypervisorType.KVM || @@ -75,8 +76,11 @@ public class StoragePoolMonitor implements Listener { for (StoragePoolVO pool : pools) { Long hostId = host.getId(); s_logger.debug("Host " + hostId + " connected, sending down storage pool information ..."); - if(_storageManager.addPoolToHost(hostId, pool)){ + try { + _storageManager.addPoolToHost(hostId, pool); _storageManager.createCapacityEntry(pool); + } catch (Exception e) { + throw new ConnectionException(true, "Unable to connect to pool " + pool, e); } } }