From 43c8da2d0e8466a45b9de31221c17748003d6a0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gabriel=20Beims=20Br=C3=A4scher?= Date: Sun, 8 Aug 2021 16:38:06 -0300 Subject: [PATCH] API-call to declare host as Degraded (#4111) * Declare host as dead * Enhance DeclareHostAsDeadCmd and add CancelHostAsDeadCmd * Stop VMs on Dead Host - Enhance code * Add "since" on API, enhance description; change API cmds response handling * Replace the ResourceState and command names from 'Dead' to 'Degraded' * Replace missing 'Dead' word to 'Degraded' * Update API version for 4.16.0.0 --- .../main/java/com/cloud/event/EventTypes.java | 4 + .../com/cloud/resource/ResourceService.java | 8 +- .../com/cloud/resource/ResourceState.java | 12 +- .../apache/cloudstack/api/ApiConstants.java | 1 + .../admin/host/CancelHostAsDegradedCmd.java | 113 ++++++++++++++++++ .../admin/host/DeclareHostAsDegradedCmd.java | 113 ++++++++++++++++++ .../cloud/resource/ResourceManagerImpl.java | 96 ++++++++++++++- .../cloud/server/ManagementServerImpl.java | 4 + .../OutOfBandManagementServiceImpl.java | 8 ++ .../resource/MockResourceManagerImpl.java | 13 ++ .../resource/ResourceManagerImplTest.java | 96 +++++++++++++++ 11 files changed, 464 insertions(+), 4 deletions(-) create mode 100644 api/src/main/java/org/apache/cloudstack/api/command/admin/host/CancelHostAsDegradedCmd.java create mode 100644 api/src/main/java/org/apache/cloudstack/api/command/admin/host/DeclareHostAsDegradedCmd.java diff --git a/api/src/main/java/com/cloud/event/EventTypes.java b/api/src/main/java/com/cloud/event/EventTypes.java index 025d9e97d95..5947334016c 100644 --- a/api/src/main/java/com/cloud/event/EventTypes.java +++ b/api/src/main/java/com/cloud/event/EventTypes.java @@ -351,6 +351,10 @@ public class EventTypes { // Host public static final String EVENT_HOST_RECONNECT = "HOST.RECONNECT"; + // Host on Degraded ResourceState + public static final String EVENT_DECLARE_HOST_DEGRADED = "HOST.DECLARE.DEGRADED"; + public static final String EVENT_CANCEL_HOST_DEGRADED = "HOST.CANCEL.DEGRADED"; + // Host Out-of-band management public static final String EVENT_HOST_OUTOFBAND_MANAGEMENT_ENABLE = "HOST.OOBM.ENABLE"; public static final String EVENT_HOST_OUTOFBAND_MANAGEMENT_DISABLE = "HOST.OOBM.DISABLE"; diff --git a/api/src/main/java/com/cloud/resource/ResourceService.java b/api/src/main/java/com/cloud/resource/ResourceService.java index 7f04d8919b9..e2b84ba8720 100644 --- a/api/src/main/java/com/cloud/resource/ResourceService.java +++ b/api/src/main/java/com/cloud/resource/ResourceService.java @@ -24,10 +24,12 @@ import org.apache.cloudstack.api.command.admin.cluster.UpdateClusterCmd; import org.apache.cloudstack.api.command.admin.host.AddHostCmd; import org.apache.cloudstack.api.command.admin.host.AddSecondaryStorageCmd; import org.apache.cloudstack.api.command.admin.host.CancelMaintenanceCmd; -import org.apache.cloudstack.api.command.admin.host.PrepareForMaintenanceCmd; import org.apache.cloudstack.api.command.admin.host.ReconnectHostCmd; import org.apache.cloudstack.api.command.admin.host.UpdateHostCmd; import org.apache.cloudstack.api.command.admin.host.UpdateHostPasswordCmd; +import org.apache.cloudstack.api.command.admin.host.PrepareForMaintenanceCmd; +import org.apache.cloudstack.api.command.admin.host.DeclareHostAsDegradedCmd; +import org.apache.cloudstack.api.command.admin.host.CancelHostAsDegradedCmd; import com.cloud.dc.DataCenter; import com.cloud.exception.AgentUnavailableException; @@ -67,6 +69,10 @@ public interface ResourceService { Host maintain(PrepareForMaintenanceCmd cmd); + Host declareHostAsDegraded(DeclareHostAsDegradedCmd cmd) throws NoTransitionException; + + Host cancelHostAsDegraded(CancelHostAsDegradedCmd cmd) throws NoTransitionException; + /** * Deletes a host * @param true if deleted, false otherwise diff --git a/api/src/main/java/com/cloud/resource/ResourceState.java b/api/src/main/java/com/cloud/resource/ResourceState.java index 6e0fa909230..70738c7921b 100644 --- a/api/src/main/java/com/cloud/resource/ResourceState.java +++ b/api/src/main/java/com/cloud/resource/ResourceState.java @@ -30,7 +30,8 @@ public enum ResourceState { PrepareForMaintenance, ErrorInMaintenance, Maintenance, - Error; + Error, + Degraded; public enum Event { InternalCreated("Resource is created"), @@ -45,6 +46,8 @@ public enum ResourceState { ErrorsCorrected("Errors were corrected on a resource attempting to enter maintenance but encountered errors"), Error("An internal error happened"), DeleteHost("Admin delete a host"), + DeclareHostDegraded("Admin declares host as Degraded"), + EnableDegradedHost("Admin puts Degraded host into Enabled"), /* * Below events don't cause resource state to change, they are merely @@ -113,11 +116,13 @@ public enum ResourceState { s_fsm.addTransition(ResourceState.Enabled, Event.InternalCreated, ResourceState.Enabled); s_fsm.addTransition(ResourceState.Enabled, Event.Disable, ResourceState.Disabled); s_fsm.addTransition(ResourceState.Enabled, Event.AdminAskMaintenance, ResourceState.PrepareForMaintenance); + s_fsm.addTransition(ResourceState.Enabled, Event.DeclareHostDegraded, ResourceState.Degraded); s_fsm.addTransition(ResourceState.Enabled, Event.InternalEnterMaintenance, ResourceState.Maintenance); s_fsm.addTransition(ResourceState.Enabled, Event.DeleteHost, ResourceState.Disabled); s_fsm.addTransition(ResourceState.Disabled, Event.Enable, ResourceState.Enabled); s_fsm.addTransition(ResourceState.Disabled, Event.Disable, ResourceState.Disabled); s_fsm.addTransition(ResourceState.Disabled, Event.InternalCreated, ResourceState.Disabled); + s_fsm.addTransition(ResourceState.Disabled, Event.DeclareHostDegraded, ResourceState.Degraded); s_fsm.addTransition(ResourceState.PrepareForMaintenance, Event.InternalEnterMaintenance, ResourceState.Maintenance); s_fsm.addTransition(ResourceState.PrepareForMaintenance, Event.AdminCancelMaintenance, ResourceState.Enabled); s_fsm.addTransition(ResourceState.PrepareForMaintenance, Event.UnableToMigrate, ResourceState.ErrorInPrepareForMaintenance); @@ -126,6 +131,7 @@ public enum ResourceState { s_fsm.addTransition(ResourceState.Maintenance, Event.AdminCancelMaintenance, ResourceState.Enabled); s_fsm.addTransition(ResourceState.Maintenance, Event.InternalCreated, ResourceState.Maintenance); s_fsm.addTransition(ResourceState.Maintenance, Event.DeleteHost, ResourceState.Disabled); + s_fsm.addTransition(ResourceState.Maintenance, Event.DeclareHostDegraded, ResourceState.Degraded); s_fsm.addTransition(ResourceState.ErrorInPrepareForMaintenance, Event.InternalCreated, ResourceState.ErrorInPrepareForMaintenance); s_fsm.addTransition(ResourceState.ErrorInPrepareForMaintenance, Event.Disable, ResourceState.Disabled); s_fsm.addTransition(ResourceState.ErrorInPrepareForMaintenance, Event.DeleteHost, ResourceState.Disabled); @@ -141,6 +147,8 @@ public enum ResourceState { s_fsm.addTransition(ResourceState.ErrorInMaintenance, Event.AdminCancelMaintenance, ResourceState.Enabled); s_fsm.addTransition(ResourceState.Error, Event.InternalCreated, ResourceState.Error); s_fsm.addTransition(ResourceState.Disabled, Event.DeleteHost, ResourceState.Disabled); - + s_fsm.addTransition(ResourceState.Degraded, Event.DeleteHost, ResourceState.Disabled); + s_fsm.addTransition(ResourceState.Degraded, Event.EnableDegradedHost, ResourceState.Enabled); + s_fsm.addTransition(ResourceState.Degraded, Event.AdminAskMaintenance, ResourceState.Maintenance); } } diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 6d64fe9778b..e3d681f235c 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -156,6 +156,7 @@ public class ApiConstants { public static final String FIRSTNAME = "firstname"; public static final String FORCED = "forced"; public static final String FORCED_DESTROY_LOCAL_STORAGE = "forcedestroylocalstorage"; + public static final String FORCE_DELETE_HOST = "forcedeletehost"; public static final String FORMAT = "format"; public static final String FOR_VIRTUAL_NETWORK = "forvirtualnetwork"; public static final String FOR_SYSTEM_VMS = "forsystemvms"; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/host/CancelHostAsDegradedCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/host/CancelHostAsDegradedCmd.java new file mode 100644 index 00000000000..98557dd710a --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/host/CancelHostAsDegradedCmd.java @@ -0,0 +1,113 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.admin.host; + +import com.cloud.event.EventTypes; +import com.cloud.host.Host; +import com.cloud.utils.fsm.NoTransitionException; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.ApiArgValidator; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandJobType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.BaseAsyncCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.HostResponse; +import org.apache.cloudstack.context.CallContext; + +@APICommand(name = "cancelHostAsDegraded", + description = "Cancel host status from 'Degraded'. Host will transit back to status 'Enabled'.", + since = "4.16.0.0", + responseObject = HostResponse.class, + requestHasSensitiveInfo = false, + responseHasSensitiveInfo = false, + authorized = {RoleType.Admin}) +public class CancelHostAsDegradedCmd extends BaseAsyncCmd { + + private static final String COMMAND_RESPONSE_NAME = "cancelhostasdegradedresponse"; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.ID, type = BaseCmd.CommandType.UUID, entityType = HostResponse.class, description = "host ID", required = true, validations = {ApiArgValidator.PositiveNumber}) + private Long id; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public Long getId() { + return id; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public String getCommandName() { + return COMMAND_RESPONSE_NAME; + } + + public static String getResultObjectName() { + return "host"; + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccountId(); + } + + @Override + public String getEventType() { + return EventTypes.EVENT_CANCEL_HOST_DEGRADED; + } + + @Override + public String getEventDescription() { + return "declaring host: " + getId() + " as Degraded"; + } + + @Override + public ApiCommandJobType getInstanceType() { + return ApiCommandJobType.Host; + } + + @Override + public Long getInstanceId() { + return getId(); + } + + @Override + public void execute() { + Host host; + try { + host = _resourceService.cancelHostAsDegraded(this); + } catch (NoTransitionException exception) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to Cancel host from Degraded status due to: " + exception.getMessage()); + } + + HostResponse response = _responseGenerator.createHostResponse(host); + response.setResponseName(COMMAND_RESPONSE_NAME); + this.setResponseObject(response); + } + +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/host/DeclareHostAsDegradedCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/host/DeclareHostAsDegradedCmd.java new file mode 100644 index 00000000000..bdf440fc054 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/host/DeclareHostAsDegradedCmd.java @@ -0,0 +1,113 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.admin.host; + +import com.cloud.event.EventTypes; +import com.cloud.host.Host; +import com.cloud.utils.fsm.NoTransitionException; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.ApiArgValidator; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandJobType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.BaseAsyncCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.HostResponse; +import org.apache.cloudstack.context.CallContext; + +@APICommand(name = "declareHostAsDegraded", + description = "Declare host as 'Degraded'. Host must be on 'Disconnected' or 'Alert' state. The ADMIN must be sure that there are no VMs running on the respective host otherwise this command might corrupted VMs that were running on the 'Degraded' host.", + since = "4.16.0.0", + responseObject = HostResponse.class, + requestHasSensitiveInfo = false, + responseHasSensitiveInfo = false, + authorized = {RoleType.Admin}) +public class DeclareHostAsDegradedCmd extends BaseAsyncCmd { + + private static final String COMMAND_RESPONSE_NAME = "declarehostasdegradedresponse"; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.ID, type = BaseCmd.CommandType.UUID, entityType = HostResponse.class, description = "host ID", required = true, validations = {ApiArgValidator.PositiveNumber}) + private Long id; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public Long getId() { + return id; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public String getCommandName() { + return COMMAND_RESPONSE_NAME; + } + + public static String getResultObjectName() { + return "host"; + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccountId(); + } + + @Override + public String getEventType() { + return EventTypes.EVENT_DECLARE_HOST_DEGRADED; + } + + @Override + public String getEventDescription() { + return "declaring host: " + getId() + " as Degraded"; + } + + @Override + public ApiCommandJobType getInstanceType() { + return ApiCommandJobType.Host; + } + + @Override + public Long getInstanceId() { + return getId(); + } + + @Override + public void execute() { + Host host; + try { + host = _resourceService.declareHostAsDegraded(this); + } catch (NoTransitionException exception) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to declare host as Degraded due to: " + exception.getMessage()); + } + + HostResponse response = _responseGenerator.createHostResponse(host); + response.setResponseName(COMMAND_RESPONSE_NAME); + this.setResponseObject(response); + } + +} diff --git a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java index 488be99e32d..ad0190ffbad 100755 --- a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java +++ b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java @@ -50,11 +50,14 @@ import org.apache.cloudstack.api.command.admin.cluster.DeleteClusterCmd; import org.apache.cloudstack.api.command.admin.cluster.UpdateClusterCmd; import org.apache.cloudstack.api.command.admin.host.AddHostCmd; import org.apache.cloudstack.api.command.admin.host.AddSecondaryStorageCmd; +import org.apache.cloudstack.api.command.admin.host.CancelHostAsDegradedCmd; import org.apache.cloudstack.api.command.admin.host.CancelMaintenanceCmd; import org.apache.cloudstack.api.command.admin.host.PrepareForMaintenanceCmd; +import org.apache.cloudstack.api.command.admin.host.DeclareHostAsDegradedCmd; import org.apache.cloudstack.api.command.admin.host.ReconnectHostCmd; import org.apache.cloudstack.api.command.admin.host.UpdateHostCmd; import org.apache.cloudstack.api.command.admin.host.UpdateHostPasswordCmd; + import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; @@ -860,7 +863,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, } _accountMgr.checkAccessAndSpecifyAuthority(CallContext.current().getCallingAccount(), host.getDataCenterId()); - if (!isForced && host.getResourceState() != ResourceState.Maintenance) { + if (!canDeleteHost(host) && !isForced) { throw new CloudRuntimeException("Host " + host.getUuid() + " cannot be deleted as it is not in maintenance mode. Either put the host into maintenance or perform a forced deletion."); } @@ -973,6 +976,14 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, return true; } + /** + * Returns true if host can be deleted.
+ * A host can be deleted either if it is in Maintenance or "Degraded" state. + */ + protected boolean canDeleteHost(HostVO host) { + return host.getResourceState() == ResourceState.Maintenance || host.getResourceState() == ResourceState.Degraded; + } + @Override public boolean deleteHost(final long hostId, final boolean isForced, final boolean isForceDeleteStorage) { try { @@ -1448,6 +1459,89 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, return false; } + /** + * Declares host as Degraded. This method is used in critical situations; e.g. if it is not possible to start host, not even via out-of-band. + */ + @Override + public Host declareHostAsDegraded(final DeclareHostAsDegradedCmd cmd) throws NoTransitionException { + Long hostId = cmd.getId(); + HostVO host = _hostDao.findById(hostId); + + if (host == null || StringUtils.isBlank(host.getName())) { + throw new InvalidParameterValueException(String.format("Host [id:%s] does not exist.", hostId)); + } else if (host.getRemoved() != null){ + throw new InvalidParameterValueException(String.format("Host [id:%s, name:%s] does not exist or it has been removed.", hostId, host.getName())); + } + + if (host.getResourceState() == ResourceState.Degraded) { + throw new NoTransitionException(String.format("Host [id:%s] was already marked as Degraded.", host.getId())); + } + + if (host.getStatus() != Status.Alert && host.getStatus() != Status.Disconnected) { + throw new InvalidParameterValueException( + String.format("Cannot perform declare host [id=%s, name=%s] as 'Degraded' when host is in %s status", host.getId(), host.getName(), host.getStatus())); + } + + try { + resourceStateTransitTo(host, ResourceState.Event.DeclareHostDegraded, _nodeId); + host.setResourceState(ResourceState.Degraded); + } catch (NoTransitionException e) { + s_logger.error(String.format("Cannot transmit host [id:%s, name:%s, state:%s, status:%s] to %s state", host.getId(), host.getName(), host.getState(), host.getStatus(), + ResourceState.Event.DeclareHostDegraded), e); + throw e; + } + + scheduleVmsRestart(hostId); + + return host; + } + + /** + * This method assumes that the host is Degraded; therefore it schedule VMs to be re-started by the HA manager. + */ + private void scheduleVmsRestart(Long hostId) { + List allVmsOnHost = _vmDao.listByHostId(hostId); + if (CollectionUtils.isEmpty(allVmsOnHost)) { + s_logger.debug(String.format("Host [id=%s] was marked as Degraded with no allocated VMs, no need to schedule VM restart", hostId)); + } + + s_logger.debug(String.format("Host [id=%s] was marked as Degraded with a total of %s allocated VMs. Triggering HA to start VMs that have HA enabled.", hostId, allVmsOnHost.size())); + for (VMInstanceVO vm : allVmsOnHost) { + State vmState = vm.getState(); + if (vmState == State.Starting || vmState == State.Running || vmState == State.Stopping) { + _haMgr.scheduleRestart(vm, false); + } + } + } + + /** + * Changes a host from 'Degraded' to 'Enabled' ResourceState. + */ + @Override + public Host cancelHostAsDegraded(final CancelHostAsDegradedCmd cmd) throws NoTransitionException { + Long hostId = cmd.getId(); + HostVO host = _hostDao.findById(hostId); + + if (host == null || host.getRemoved() != null) { + throw new InvalidParameterValueException(String.format("Host [id=%s] does not exist", host.getId())); + } + + if (host.getResourceState() != ResourceState.Degraded) { + throw new NoTransitionException( + String.format("Cannot perform cancelHostAsDegraded on host [id=%s, name=%s] when host is in %s state", host.getId(), host.getName(), host.getResourceState())); + } + + try { + resourceStateTransitTo(host, ResourceState.Event.EnableDegradedHost, _nodeId); + host.setResourceState(ResourceState.Enabled); + } catch (NoTransitionException e) { + throw new NoTransitionException( + String.format("Cannot transmit host [id=%s, name=%s, state=%s, status=%s] to %s state", host.getId(), host.getName(), host.getResourceState(), host.getStatus(), + ResourceState.Enabled)); + } + return host; + } + /** * Add VNC details as user VM details for each VM in 'vms' (KVM hosts only) */ diff --git a/server/src/main/java/com/cloud/server/ManagementServerImpl.java b/server/src/main/java/com/cloud/server/ManagementServerImpl.java index 607263d2f4f..34e9a880cd2 100644 --- a/server/src/main/java/com/cloud/server/ManagementServerImpl.java +++ b/server/src/main/java/com/cloud/server/ManagementServerImpl.java @@ -89,7 +89,9 @@ import org.apache.cloudstack.api.command.admin.guest.UpdateGuestOsCmd; import org.apache.cloudstack.api.command.admin.guest.UpdateGuestOsMappingCmd; import org.apache.cloudstack.api.command.admin.host.AddHostCmd; import org.apache.cloudstack.api.command.admin.host.AddSecondaryStorageCmd; +import org.apache.cloudstack.api.command.admin.host.CancelHostAsDegradedCmd; import org.apache.cloudstack.api.command.admin.host.CancelMaintenanceCmd; +import org.apache.cloudstack.api.command.admin.host.DeclareHostAsDegradedCmd; import org.apache.cloudstack.api.command.admin.host.DeleteHostCmd; import org.apache.cloudstack.api.command.admin.host.FindHostsForMigrationCmd; import org.apache.cloudstack.api.command.admin.host.ListHostTagsCmd; @@ -2975,6 +2977,8 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe cmdList.add(AddHostCmd.class); cmdList.add(AddSecondaryStorageCmd.class); cmdList.add(CancelMaintenanceCmd.class); + cmdList.add(CancelHostAsDegradedCmd.class); + cmdList.add(DeclareHostAsDegradedCmd.class); cmdList.add(DeleteHostCmd.class); cmdList.add(ListHostsCmd.class); cmdList.add(ListHostTagsCmd.class); diff --git a/server/src/main/java/org/apache/cloudstack/outofbandmanagement/OutOfBandManagementServiceImpl.java b/server/src/main/java/org/apache/cloudstack/outofbandmanagement/OutOfBandManagementServiceImpl.java index a956cf6727b..e4e221daca2 100644 --- a/server/src/main/java/org/apache/cloudstack/outofbandmanagement/OutOfBandManagementServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/outofbandmanagement/OutOfBandManagementServiceImpl.java @@ -29,6 +29,7 @@ import com.cloud.event.EventTypes; import com.cloud.host.Host; import com.cloud.host.dao.HostDao; import com.cloud.org.Cluster; +import com.cloud.resource.ResourceState; import com.cloud.utils.component.Manager; import com.cloud.utils.component.ManagerBase; import com.cloud.utils.db.Transaction; @@ -248,6 +249,13 @@ public class OutOfBandManagementServiceImpl extends ManagerBase implements OutOf if (hostId == null) { return false; } + + Host host = hostDao.findById(hostId); + if (host == null || host.getResourceState() == ResourceState.Degraded) { + LOG.debug(String.format("Host [id=%s, state=] was removed or placed in Degraded state by the Admin.", hostId, host.getResourceState())); + return false; + } + final OutOfBandManagement outOfBandManagementConfig = outOfBandManagementDao.findByHost(hostId); if (outOfBandManagementConfig == null || !outOfBandManagementConfig.isEnabled()) { return false; diff --git a/server/src/test/java/com/cloud/resource/MockResourceManagerImpl.java b/server/src/test/java/com/cloud/resource/MockResourceManagerImpl.java index 4e1daa87c34..4d5b5ba584b 100755 --- a/server/src/test/java/com/cloud/resource/MockResourceManagerImpl.java +++ b/server/src/test/java/com/cloud/resource/MockResourceManagerImpl.java @@ -33,6 +33,9 @@ import org.apache.cloudstack.api.command.admin.host.PrepareForMaintenanceCmd; import org.apache.cloudstack.api.command.admin.host.ReconnectHostCmd; import org.apache.cloudstack.api.command.admin.host.UpdateHostCmd; import org.apache.cloudstack.api.command.admin.host.UpdateHostPasswordCmd; +import org.apache.cloudstack.api.command.admin.host.CancelHostAsDegradedCmd; +import org.apache.cloudstack.api.command.admin.host.DeclareHostAsDegradedCmd; + import org.apache.cloudstack.framework.config.ConfigKey; import com.cloud.agent.api.StartupCommand; @@ -142,6 +145,16 @@ public class MockResourceManagerImpl extends ManagerBase implements ResourceMana return null; } + @Override + public Host declareHostAsDegraded(DeclareHostAsDegradedCmd cmd) { + return null; + } + + @Override + public Host cancelHostAsDegraded(final CancelHostAsDegradedCmd cmd) { + return null; + } + @Override public boolean updateClusterPassword(final UpdateHostPasswordCmd upasscmd) { // TODO Auto-generated method stub diff --git a/server/src/test/java/com/cloud/resource/ResourceManagerImplTest.java b/server/src/test/java/com/cloud/resource/ResourceManagerImplTest.java index 6faa83bc910..abc03ad3d70 100644 --- a/server/src/test/java/com/cloud/resource/ResourceManagerImplTest.java +++ b/server/src/test/java/com/cloud/resource/ResourceManagerImplTest.java @@ -35,7 +35,11 @@ import static org.mockito.Mockito.when; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.UUID; +import com.cloud.exception.InvalidParameterValueException; +import org.apache.cloudstack.api.command.admin.host.CancelHostAsDegradedCmd; +import org.apache.cloudstack.api.command.admin.host.DeclareHostAsDegradedCmd; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.junit.Assert; import org.junit.Before; @@ -46,6 +50,7 @@ import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.MockitoAnnotations; import org.mockito.Spy; +import org.mockito.Mockito; import org.powermock.api.mockito.PowerMockito; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; @@ -427,4 +432,95 @@ public class ResourceManagerImplTest { verify(resourceManager, never()).resourceStateTransitTo(anyObject(), any(), anyLong()); Assert.assertFalse(enterMaintenanceMode); } + + @Test + public void declareHostAsDegradedTestDisconnected() throws NoTransitionException { + prepareAndTestDeclareHostAsDegraded(Status.Disconnected, ResourceState.Enabled, ResourceState.Degraded); + } + + @Test + public void declareHostAsDegradedTestAlert() throws NoTransitionException { + prepareAndTestDeclareHostAsDegraded(Status.Alert, ResourceState.Enabled, ResourceState.Degraded); + } + + @Test(expected = InvalidParameterValueException.class) + public void declareHostAsDegradedExpectNoTransitionException() throws NoTransitionException { + Status[] statusArray = Status.values(); + for (int i = 0; i < statusArray.length - 1; i++) { + if (statusArray[i] != Status.Alert && statusArray[i] != Status.Disconnected) { + prepareAndTestDeclareHostAsDegraded(statusArray[i], ResourceState.Enabled, ResourceState.Enabled); + } + } + } + + @Test(expected = NoTransitionException.class) + public void declareHostAsDegradedTestAlreadyDegraded() throws NoTransitionException { + prepareAndTestDeclareHostAsDegraded(Status.Alert, ResourceState.Degraded, ResourceState.Degraded); + } + + @Test(expected = NoTransitionException.class) + public void declareHostAsDegradedTestOnError() throws NoTransitionException { + prepareAndTestDeclareHostAsDegraded(Status.Alert, ResourceState.Error, ResourceState.Degraded); + } + + @Test(expected = NoTransitionException.class) + public void declareHostAsDegradedTestOnCreating() throws NoTransitionException { + prepareAndTestDeclareHostAsDegraded(Status.Alert, ResourceState.Creating, ResourceState.Degraded); + } + + @Test(expected = NoTransitionException.class) + public void declareHostAsDegradedTestOnErrorInMaintenance() throws NoTransitionException { + prepareAndTestDeclareHostAsDegraded(Status.Alert, ResourceState.ErrorInPrepareForMaintenance, ResourceState.Degraded); + } + + @Test + public void declareHostAsDegradedTestSupportedStates() throws NoTransitionException { + ResourceState[] states = ResourceState.values(); + for (int i = 0; i < states.length - 1; i++) { + if (states[i] == ResourceState.Enabled + || states[i] == ResourceState.Maintenance + || states[i] == ResourceState.Disabled) { + prepareAndTestDeclareHostAsDegraded(Status.Alert, states[i], ResourceState.Degraded); + } + } + } + + private void prepareAndTestDeclareHostAsDegraded(Status hostStatus, ResourceState originalState, ResourceState expectedResourceState) throws NoTransitionException { + DeclareHostAsDegradedCmd declareHostAsDegradedCmd = Mockito.spy(new DeclareHostAsDegradedCmd()); + HostVO hostVo = createDummyHost(hostStatus); + hostVo.setResourceState(originalState); + when(declareHostAsDegradedCmd.getId()).thenReturn(0l); + when(hostDao.findById(0l)).thenReturn(hostVo); + + Host result = resourceManager.declareHostAsDegraded(declareHostAsDegradedCmd); + + Assert.assertEquals(expectedResourceState, hostVo.getResourceState()); + } + + @Test + public void cancelHostAsDegradedTest() throws NoTransitionException { + prepareAndTestCancelHostAsDegraded(Status.Alert, ResourceState.Degraded, ResourceState.Enabled); + } + + @Test(expected = NoTransitionException.class) + public void cancelHostAsDegradedTestHostNotDegraded() throws NoTransitionException { + prepareAndTestCancelHostAsDegraded(Status.Alert, ResourceState.Enabled, ResourceState.Enabled); + } + + private void prepareAndTestCancelHostAsDegraded(Status hostStatus, ResourceState originalState, ResourceState expectedResourceState) throws NoTransitionException { + CancelHostAsDegradedCmd cancelHostAsDegradedCmd = Mockito.spy(new CancelHostAsDegradedCmd()); + HostVO hostVo = createDummyHost(hostStatus); + hostVo.setResourceState(originalState); + when(cancelHostAsDegradedCmd.getId()).thenReturn(0l); + when(hostDao.findById(0l)).thenReturn(hostVo); + + Host result = resourceManager.cancelHostAsDegraded(cancelHostAsDegradedCmd); + + Assert.assertEquals(expectedResourceState, hostVo.getResourceState()); + } + + private HostVO createDummyHost(Status hostStatus) { + return new HostVO(1L, "host01", Host.Type.Routing, "192.168.1.1", "255.255.255.0", null, null, null, null, null, null, null, null, null, null, UUID.randomUUID().toString(), + hostStatus, "1.0", null, null, 1L, null, 0, 0, null, 0, null); + } }