etan39
/
cloudstack
mirror of https://github.com/apache/cloudstack.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

APPLE-165: Host HA management and HA provider for KVM 

Host-HA offers investigation, fencing and recovery mechanisms for host that for
any reason are malfunctioning. It uses Activity and Health checks to determine
current host state based on which it may degrade a host or try to recover it. On
failing to recover it, it may try to fence the host.

The core feature is implemented in a hypervisor agnostic way, with two separate
implementations of the driver/provider for Simulator and KVM hypervisors. The
framework also allows for implementation of other hypervisor specific provider
implementation in future.

The Host-HA provider implementation for KVM hypervisor uses the out-of-band
management sub-system to issue IPMI calls to reset (recover) or poweroff (fence)
a host.

The Host-HA provider implementation for Simulator provides a means of testing
and validating the core framework implementation.

Signed-off-by: Rohit Yadav <rohit.yadav@shapeblue.com>
This commit is contained in:
Rohit Yadav 2016-12-27 13:51:58 +05:30
parent 2b4cdd6580
commit 876fc7434d
120 changed files with 8467 additions and 420 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
api/src/com/cloud/dc/DataCenter.java
View File

@ -18,15 +18,14 @@ package com.cloud.dc;
import com.cloud.org.Grouping;
import org.apache.cloudstack.acl.InfrastructureEntity;
import org.apache.cloudstack.api.Identity;
import org.apache.cloudstack.api.InternalIdentity;
import org.apache.cloudstack.kernel.Partition;
import java.util.Map;
/**
*
*/
public interface DataCenter extends InfrastructureEntity, Grouping, Identity, InternalIdentity {
public interface DataCenter extends InfrastructureEntity, Grouping, Partition {
public enum NetworkType {
Basic, Advanced,

13
api/src/com/cloud/event/EventTypes.java
View File

@ -74,6 +74,7 @@ import com.cloud.template.VirtualMachineTemplate;
import com.cloud.user.Account;
import com.cloud.user.User;
import com.cloud.vm.VirtualMachine;
import org.apache.cloudstack.ha.HAConfig;
public class EventTypes {
@ -318,6 +319,12 @@ public class EventTypes {
public static final String EVENT_HOST_OUTOFBAND_MANAGEMENT_CHANGE_PASSWORD = "HOST.OOBM.CHANGEPASSWORD";
public static final String EVENT_HOST_OUTOFBAND_MANAGEMENT_POWERSTATE_TRANSITION = "HOST.OOBM.POWERSTATE.TRANSITION";
// HA
public static final String EVENT_HA_RESOURCE_ENABLE = "HA.RESOURCE.ENABLE";
public static final String EVENT_HA_RESOURCE_DISABLE = "HA.RESOURCE.DISABLE";
public static final String EVENT_HA_RESOURCE_CONFIGURE = "HA.RESOURCE.CONFIGURE";
public static final String EVENT_HA_STATE_TRANSITION = "HA.STATE.TRANSITION";
// Maintenance
public static final String EVENT_MAINTENANCE_CANCEL = "MAINT.CANCEL";
public static final String EVENT_MAINTENANCE_CANCEL_PRIMARY_STORAGE = "MAINT.CANCEL.PS";
@ -754,6 +761,12 @@ public class EventTypes {
entityEventDetails.put(EVENT_HOST_OUTOFBAND_MANAGEMENT_CHANGE_PASSWORD, Host.class);
entityEventDetails.put(EVENT_HOST_OUTOFBAND_MANAGEMENT_POWERSTATE_TRANSITION, Host.class);
// HA
entityEventDetails.put(EVENT_HA_RESOURCE_ENABLE, HAConfig.class);
entityEventDetails.put(EVENT_HA_RESOURCE_DISABLE, HAConfig.class);
entityEventDetails.put(EVENT_HA_RESOURCE_CONFIGURE, HAConfig.class);
entityEventDetails.put(EVENT_HA_STATE_TRANSITION, HAConfig.class);
// Maintenance
entityEventDetails.put(EVENT_MAINTENANCE_CANCEL, Host.class);
entityEventDetails.put(EVENT_MAINTENANCE_CANCEL_PRIMARY_STORAGE, Host.class);

14
api/src/com/cloud/host/Host.java
View File

@ -16,19 +16,19 @@
// under the License.
package com.cloud.host;
import java.util.Date;
import org.apache.cloudstack.api.Identity;
import org.apache.cloudstack.api.InternalIdentity;
import com.cloud.hypervisor.Hypervisor.HypervisorType;
import com.cloud.resource.ResourceState;
import com.cloud.utils.fsm.StateObject;
import org.apache.cloudstack.api.Identity;
import org.apache.cloudstack.ha.HAResource;
import org.apache.cloudstack.kernel.Partition;
import java.util.Date;
/**
* Host represents one particular host server.
*/
public interface Host extends StateObject<Status>, Identity, InternalIdentity {
public interface Host extends StateObject<Status>, Identity, Partition, HAResource {
public enum Type {
Storage(false), Routing(false), SecondaryStorage(false), SecondaryStorageCmdExecutor(false), ConsoleProxy(true), ExternalFirewall(false), ExternalLoadBalancer(
false), ExternalVirtualSwitchSupervisor(false), PxeServer(false), BaremetalPxe(false), BaremetalDhcp(false), TrafficMonitor(false),
@ -202,5 +202,7 @@ public interface Host extends StateObject<Status>, Identity, InternalIdentity {
boolean isInMaintenanceStates();
boolean isDisabled();
ResourceState getResourceState();
}

2
api/src/com/cloud/host/Status.java
View File

@ -150,12 +150,14 @@ public enum Status {
s_fsm.addTransition(Status.Down, Event.ManagementServerDown, Status.Down);
s_fsm.addTransition(Status.Down, Event.AgentDisconnected, Status.Down);
s_fsm.addTransition(Status.Down, Event.PingTimeout, Status.Down);
s_fsm.addTransition(Status.Down, Event.HostDown, Status.Down);
s_fsm.addTransition(Status.Alert, Event.AgentConnected, Status.Connecting);
s_fsm.addTransition(Status.Alert, Event.Ping, Status.Up);
s_fsm.addTransition(Status.Alert, Event.Remove, Status.Removed);
s_fsm.addTransition(Status.Alert, Event.ManagementServerDown, Status.Alert);
s_fsm.addTransition(Status.Alert, Event.AgentDisconnected, Status.Alert);
s_fsm.addTransition(Status.Alert, Event.ShutdownRequested, Status.Disconnected);
s_fsm.addTransition(Status.Alert, Event.HostDown, Status.Down);
s_fsm.addTransition(Status.Rebalancing, Event.RebalanceFailed, Status.Disconnected);
s_fsm.addTransition(Status.Rebalancing, Event.RebalanceCompleted, Status.Connecting);
s_fsm.addTransition(Status.Rebalancing, Event.ManagementServerDown, Status.Disconnected);

6
api/src/com/cloud/org/Cluster.java
View File

@ -16,13 +16,11 @@
// under the License.
package com.cloud.org;
import org.apache.cloudstack.api.Identity;
import org.apache.cloudstack.api.InternalIdentity;
import com.cloud.hypervisor.Hypervisor.HypervisorType;
import com.cloud.org.Managed.ManagedState;
import org.apache.cloudstack.kernel.Partition;
public interface Cluster extends Grouping, InternalIdentity, Identity {
public interface Cluster extends Grouping, Partition {
public static enum ClusterType {
CloudManaged, ExternalManaged;
};

3
api/src/com/cloud/resource/ResourceState.java
View File

@ -93,6 +93,7 @@ public enum ResourceState {
s_fsm.addTransition(ResourceState.Enabled, Event.InternalCreated, ResourceState.Enabled);
s_fsm.addTransition(ResourceState.Enabled, Event.Disable, ResourceState.Disabled);
s_fsm.addTransition(ResourceState.Enabled, Event.AdminAskMaintenace, ResourceState.PrepareForMaintenance);
s_fsm.addTransition(ResourceState.Enabled, Event.InternalEnterMaintenance, ResourceState.Maintenance);
s_fsm.addTransition(ResourceState.Disabled, Event.Enable, ResourceState.Enabled);
s_fsm.addTransition(ResourceState.Disabled, Event.Disable, ResourceState.Disabled);
s_fsm.addTransition(ResourceState.Disabled, Event.InternalCreated, ResourceState.Disabled);
@ -109,5 +110,7 @@ public enum ResourceState {
s_fsm.addTransition(ResourceState.ErrorInMaintenance, Event.InternalEnterMaintenance, ResourceState.Maintenance);
s_fsm.addTransition(ResourceState.ErrorInMaintenance, Event.AdminCancelMaintenance, ResourceState.Enabled);
s_fsm.addTransition(ResourceState.Error, Event.InternalCreated, ResourceState.Error);
s_fsm.addTransition(ResourceState.Disabled, Event.DeleteHost, ResourceState.Disabled);
}
}

18
api/src/com/cloud/vm/VirtualMachine.java
View File

@ -16,26 +16,24 @@
// under the License.
package com.cloud.vm;
import java.util.Arrays;
import java.util.Date;
import java.util.Map;
import org.apache.cloudstack.acl.ControlledEntity;
import org.apache.cloudstack.api.Displayable;
import org.apache.cloudstack.api.Identity;
import org.apache.cloudstack.api.InternalIdentity;
import com.cloud.hypervisor.Hypervisor.HypervisorType;
import com.cloud.utils.fsm.StateMachine2;
import com.cloud.utils.fsm.StateMachine2.Transition;
import com.cloud.utils.fsm.StateMachine2.Transition.Impact;
import com.cloud.utils.fsm.StateObject;
import org.apache.cloudstack.acl.ControlledEntity;
import org.apache.cloudstack.api.Displayable;
import org.apache.cloudstack.kernel.Partition;
import java.util.Arrays;
import java.util.Date;
import java.util.Map;
/**
* VirtualMachine describes the properties held by a virtual machine
*
*/
public interface VirtualMachine extends RunningOn, ControlledEntity, Identity, InternalIdentity, Displayable, StateObject<VirtualMachine.State> {
public interface VirtualMachine extends RunningOn, ControlledEntity, Partition, Displayable, StateObject<VirtualMachine.State> {
public enum PowerState {
PowerUnknown,

7
api/src/org/apache/cloudstack/alert/AlertService.java
View File

@ -16,12 +16,12 @@
// under the License.
package org.apache.cloudstack.alert;
import java.util.HashSet;
import java.util.Set;
import com.cloud.capacity.Capacity;
import com.cloud.exception.InvalidParameterValueException;
import java.util.HashSet;
import java.util.Set;
public interface AlertService {
public static class AlertType {
private static Set<AlertType> defaultAlertTypes = new HashSet<AlertType>();
@ -66,6 +66,7 @@ public interface AlertService {
public static final AlertType ALERT_TYPE_RESOURCE_LIMIT_EXCEEDED = new AlertType((short)26, "ALERT.RESOURCE.EXCEED", true);
public static final AlertType ALERT_TYPE_SYNC = new AlertType((short)27, "ALERT.TYPE.SYNC", true);
public static final AlertType ALERT_TYPE_OOBM_AUTH_ERROR = new AlertType((short)29, "ALERT.OOBM.AUTHERROR", true);
public static final AlertType ALERT_TYPE_HA_ACTION = new AlertType((short)30, "ALERT.HA.ACTION", true);
public short getType() {
return type;

7
api/src/org/apache/cloudstack/api/ApiConstants.java
View File

@ -21,6 +21,7 @@ public class ApiConstants {
public static final String ACCOUNTS = "accounts";
public static final String ACCOUNT_TYPE = "accounttype";
public static final String ACCOUNT_ID = "accountid";
public static final String ACTIVITY = "activity";
public static final String ADDRESS = "address";
public static final String ALGORITHM = "algorithm";
public static final String ALLOCATED_ONLY = "allocatedonly";
@ -93,6 +94,7 @@ public class ApiConstants {
public static final String DOMAIN_ID = "domainid";
public static final String DOMAIN__ID = "domainId";
public static final String DURATION = "duration";
public static final String ELIGIBLE = "eligible";
public static final String EMAIL = "email";
public static final String END_DATE = "enddate";
public static final String END_IP = "endip";
@ -100,6 +102,7 @@ public class ApiConstants {
public static final String END_PORT = "endport";
public static final String ENTRY_TIME = "entrytime";
public static final String EXPIRES = "expires";
public static final String FENCE = "fence";
public static final String FETCH_LATEST = "fetchlatest";
public static final String FIRSTNAME = "firstname";
public static final String FORCED = "forced";
@ -118,6 +121,9 @@ public class ApiConstants {
public static final String GUEST_CIDR_ADDRESS = "guestcidraddress";
public static final String GUEST_VLAN_RANGE = "guestvlanrange";
public static final String HA_ENABLE = "haenable";
public static final String HA_PROVIDER = "haprovider";
public static final String HA_STATE = "hastate";
public static final String HEALTH = "health";
public static final String HOST_ID = "hostid";
public static final String HOST_NAME = "hostname";
public static final String HYPERVISOR = "hypervisor";
@ -212,6 +218,7 @@ public class ApiConstants {
public static final String PUBLIC_END_PORT = "publicendport";
public static final String PUBLIC_ZONE = "publiczone";
public static final String RECEIVED_BYTES = "receivedbytes";
public static final String RECOVER = "recover";
public static final String REQUIRES_HVM = "requireshvm";
public static final String RESOURCE_TYPE = "resourcetype";
public static final String RESPONSE = "response";

127
api/src/org/apache/cloudstack/api/command/admin/ha/ConfigureHAForHostCmd.java Normal file
View File

@ -0,0 +1,127 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.api.command.admin.ha;
import com.cloud.event.EventTypes;
import com.cloud.exception.ConcurrentOperationException;
import com.cloud.exception.InsufficientCapacityException;
import com.cloud.exception.NetworkRuleConflictException;
import com.cloud.exception.ResourceAllocationException;
import com.cloud.exception.ResourceUnavailableException;
import com.cloud.host.Host;
import com.cloud.user.Account;
import org.apache.cloudstack.acl.RoleType;
import org.apache.cloudstack.api.APICommand;
import org.apache.cloudstack.api.ApiArgValidator;
import org.apache.cloudstack.api.ApiConstants;
import org.apache.cloudstack.api.ApiErrorCode;
import org.apache.cloudstack.api.BaseAsyncCmd;
import org.apache.cloudstack.api.BaseCmd;
import org.apache.cloudstack.api.Parameter;
import org.apache.cloudstack.api.ServerApiException;
import org.apache.cloudstack.api.response.HostHAResponse;
import org.apache.cloudstack.api.response.HostResponse;
import org.apache.cloudstack.context.CallContext;
import org.apache.cloudstack.ha.HAConfigManager;
import org.apache.cloudstack.ha.HAResource;
import javax.inject.Inject;
@APICommand(name = ConfigureHAForHostCmd.APINAME, description = "Configures HA for a host",
responseObject = HostHAResponse.class,
requestHasSensitiveInfo = false, responseHasSensitiveInfo = false,
since = "4.11", authorized = {RoleType.Admin})
public final class ConfigureHAForHostCmd extends BaseAsyncCmd {
public static final String APINAME = "configureHAForHost";
@Inject
private HAConfigManager haConfigManager;
/////////////////////////////////////////////////////
//////////////// API parameters /////////////////////
/////////////////////////////////////////////////////
@Parameter(name = ApiConstants.HOST_ID, type = CommandType.UUID, entityType = HostResponse.class,
description = "ID of the host", required = true, validations = {ApiArgValidator.PositiveNumber})
private Long hostId;
@Parameter(name = ApiConstants.PROVIDER, type = CommandType.STRING,
description = "HA provider", required = true, validations = {ApiArgValidator.NotNullOrEmpty})
private String haProvider;
/////////////////////////////////////////////////////
/////////////////// Accessors ///////////////////////
/////////////////////////////////////////////////////
public Long getHostId() {
return hostId;
}
public String getHaProvider() {
return haProvider;
}
/////////////////////////////////////////////////////
/////////////// API Implementation///////////////////
/////////////////////////////////////////////////////
@Override
public String getCommandName() {
return APINAME.toLowerCase() + BaseCmd.RESPONSE_SUFFIX;
}
@Override
public long getEntityOwnerId() {
return Account.ACCOUNT_ID_SYSTEM;
}
private void setupResponse(final boolean result, final String resourceUuid) {
final HostHAResponse response = new HostHAResponse();
response.setId(resourceUuid);
response.setProvider(getHaProvider().toLowerCase());
response.setResponseName(getCommandName());
setResponseObject(response);
}
@Override
public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException {
final Host host = _resourceService.getHost(getHostId());
if (host == null) {
throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Unable to find host by ID: " + getHostId());
}
final boolean result = haConfigManager.configureHA(host.getId(), HAResource.ResourceType.Host, getHaProvider());
if (!result) {
throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to configure HA provider for the host");
}
CallContext.current().setEventDetails("Host Id:" + host.getId() + " HA configured with provider: " + getHaProvider());
CallContext.current().putContextParameter(Host.class, host.getUuid());
setupResponse(result, host.getUuid());
}
@Override
public String getEventType() {
return EventTypes.EVENT_HA_RESOURCE_DISABLE;
}
@Override
public String getEventDescription() {
return "configure HA for host: " + getHostId();
}
}

114
api/src/org/apache/cloudstack/api/command/admin/ha/DisableHAForClusterCmd.java Normal file
View File

@ -0,0 +1,114 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.api.command.admin.ha;
import com.cloud.event.EventTypes;
import com.cloud.exception.ConcurrentOperationException;
import com.cloud.exception.InsufficientCapacityException;
import com.cloud.exception.NetworkRuleConflictException;
import com.cloud.exception.ResourceAllocationException;
import com.cloud.exception.ResourceUnavailableException;
import com.cloud.org.Cluster;
import org.apache.cloudstack.acl.RoleType;
import org.apache.cloudstack.api.APICommand;
import org.apache.cloudstack.api.ApiArgValidator;
import org.apache.cloudstack.api.ApiConstants;
import org.apache.cloudstack.api.ApiErrorCode;
import org.apache.cloudstack.api.BaseAsyncCmd;
import org.apache.cloudstack.api.BaseCmd;
import org.apache.cloudstack.api.Parameter;
import org.apache.cloudstack.api.ServerApiException;
import org.apache.cloudstack.api.response.ClusterResponse;
import org.apache.cloudstack.api.response.SuccessResponse;
import org.apache.cloudstack.context.CallContext;
import org.apache.cloudstack.ha.HAConfigManager;
import javax.inject.Inject;
@APICommand(name = DisableHAForClusterCmd.APINAME, description = "Disables HA cluster-wide",
responseObject = SuccessResponse.class,
requestHasSensitiveInfo = false, responseHasSensitiveInfo = false,
since = "4.11", authorized = {RoleType.Admin})
public final class DisableHAForClusterCmd extends BaseAsyncCmd {
public static final String APINAME = "disableHAForCluster";
@Inject
private HAConfigManager haConfigManager;
/////////////////////////////////////////////////////
//////////////// API parameters /////////////////////
/////////////////////////////////////////////////////
@Parameter(name = ApiConstants.CLUSTER_ID, type = BaseCmd.CommandType.UUID, entityType = ClusterResponse.class,
description = "ID of the cluster", required = true, validations = {ApiArgValidator.PositiveNumber})
private Long clusterId;
/////////////////////////////////////////////////////
/////////////////// Accessors ///////////////////////
/////////////////////////////////////////////////////
public Long getClusterId() {
return clusterId;
}
/////////////////////////////////////////////////////
/////////////// API Implementation///////////////////
/////////////////////////////////////////////////////
@Override
public String getCommandName() {
return APINAME.toLowerCase() + BaseCmd.RESPONSE_SUFFIX;
}
@Override
public long getEntityOwnerId() {
return CallContext.current().getCallingAccountId();
}
private void setupResponse(final boolean result) {
final SuccessResponse response = new SuccessResponse();
response.setSuccess(result);
response.setResponseName(getCommandName());
response.setObjectName("ha");
setResponseObject(response);
}
@Override
public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException {
final Cluster cluster = _resourceService.getCluster(getClusterId());
if (cluster == null) {
throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Unable to find cluster by ID: " + getClusterId());
}
final boolean result = haConfigManager.disableHA(cluster);
CallContext.current().setEventDetails("Cluster Id:" + cluster.getId() + " HA enabled: false");
CallContext.current().putContextParameter(Cluster.class, cluster.getUuid());
setupResponse(result);
}
@Override
public String getEventType() {
return EventTypes.EVENT_HA_RESOURCE_DISABLE;
}
@Override
public String getEventDescription() {
return "disable HA for cluster: " + getClusterId();
}
}

116
api/src/org/apache/cloudstack/api/command/admin/ha/DisableHAForHostCmd.java Normal file
View File

@ -0,0 +1,116 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.api.command.admin.ha;
import com.cloud.event.EventTypes;
import com.cloud.exception.ConcurrentOperationException;
import com.cloud.exception.InsufficientCapacityException;
import com.cloud.exception.NetworkRuleConflictException;
import com.cloud.exception.ResourceAllocationException;
import com.cloud.exception.ResourceUnavailableException;
import com.cloud.host.Host;
import org.apache.cloudstack.acl.RoleType;
import org.apache.cloudstack.api.APICommand;
import org.apache.cloudstack.api.ApiArgValidator;
import org.apache.cloudstack.api.ApiConstants;
import org.apache.cloudstack.api.ApiErrorCode;
import org.apache.cloudstack.api.BaseAsyncCmd;
import org.apache.cloudstack.api.BaseCmd;
import org.apache.cloudstack.api.Parameter;
import org.apache.cloudstack.api.ServerApiException;
import org.apache.cloudstack.api.response.HostHAResponse;
import org.apache.cloudstack.api.response.HostResponse;
import org.apache.cloudstack.context.CallContext;
import org.apache.cloudstack.ha.HAConfigManager;
import org.apache.cloudstack.ha.HAResource;
import javax.inject.Inject;
@APICommand(name = DisableHAForHostCmd.APINAME, description = "Disables HA for a host",
responseObject = HostHAResponse.class,
requestHasSensitiveInfo = false, responseHasSensitiveInfo = false,
since = "4.11", authorized = {RoleType.Admin})
public final class DisableHAForHostCmd extends BaseAsyncCmd {
public static final String APINAME = "disableHAForHost";
@Inject
private HAConfigManager haConfigManager;
/////////////////////////////////////////////////////
//////////////// API parameters /////////////////////
/////////////////////////////////////////////////////
@Parameter(name = ApiConstants.HOST_ID, type = CommandType.UUID, entityType = HostResponse.class,
description = "ID of the host", required = true, validations = {ApiArgValidator.PositiveNumber})
private Long hostId;
/////////////////////////////////////////////////////
/////////////////// Accessors ///////////////////////
/////////////////////////////////////////////////////
public Long getHostId() {
return hostId;
}
/////////////////////////////////////////////////////
/////////////// API Implementation///////////////////
/////////////////////////////////////////////////////
@Override
public String getCommandName() {
return APINAME.toLowerCase() + BaseCmd.RESPONSE_SUFFIX;
}
@Override
public long getEntityOwnerId() {
return CallContext.current().getCallingAccountId();
}
private void setupResponse(final boolean result, final String resourceUuid) {
final HostHAResponse response = new HostHAResponse();
response.setId(resourceUuid);
response.setEnabled(false);
response.setStatus(result);
response.setResponseName(getCommandName());
setResponseObject(response);
}
@Override
public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException {
final Host host = _resourceService.getHost(getHostId());
if (host == null) {
throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Unable to find host by ID: " + getHostId());
}
final boolean result = haConfigManager.disableHA(host.getId(), HAResource.ResourceType.Host);
CallContext.current().setEventDetails("Host Id:" + host.getId() + " HA enabled: false");
CallContext.current().putContextParameter(Host.class, host.getUuid());
setupResponse(result, host.getUuid());
}
@Override
public String getEventType() {
return EventTypes.EVENT_HA_RESOURCE_DISABLE;
}
@Override
public String getEventDescription() {
return "disable HA for host: " + getHostId();
}
}

115
api/src/org/apache/cloudstack/api/command/admin/ha/DisableHAForZoneCmd.java Normal file
View File

@ -0,0 +1,115 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.api.command.admin.ha;
import com.cloud.dc.DataCenter;
import com.cloud.event.EventTypes;
import com.cloud.exception.ConcurrentOperationException;
import com.cloud.exception.InsufficientCapacityException;
import com.cloud.exception.NetworkRuleConflictException;
import com.cloud.exception.ResourceAllocationException;
import com.cloud.exception.ResourceUnavailableException;
import org.apache.cloudstack.acl.RoleType;
import org.apache.cloudstack.api.APICommand;
import org.apache.cloudstack.api.ApiArgValidator;
import org.apache.cloudstack.api.ApiConstants;
import org.apache.cloudstack.api.ApiErrorCode;
import org.apache.cloudstack.api.BaseAsyncCmd;
import org.apache.cloudstack.api.BaseCmd;
import org.apache.cloudstack.api.Parameter;
import org.apache.cloudstack.api.ServerApiException;
import org.apache.cloudstack.api.response.SuccessResponse;
import org.apache.cloudstack.api.response.ZoneResponse;
import org.apache.cloudstack.context.CallContext;
import org.apache.cloudstack.ha.HAConfigManager;
import javax.inject.Inject;
@APICommand(name = DisableHAForZoneCmd.APINAME, description = "Disables HA for a zone",
responseObject = SuccessResponse.class,
requestHasSensitiveInfo = false, responseHasSensitiveInfo = false,
since = "4.11", authorized = {RoleType.Admin})
public final class DisableHAForZoneCmd extends BaseAsyncCmd {
public static final String APINAME = "disableHAForZone";
@Inject
private HAConfigManager haConfigManager;
/////////////////////////////////////////////////////
//////////////// API parameters /////////////////////
/////////////////////////////////////////////////////
@Parameter(name = ApiConstants.ZONE_ID, type = BaseCmd.CommandType.UUID, entityType = ZoneResponse.class,
description = "ID of the zone", required = true, validations = {ApiArgValidator.PositiveNumber})
private Long zoneId;
/////////////////////////////////////////////////////
/////////////////// Accessors ///////////////////////
/////////////////////////////////////////////////////
public Long getZoneId() {
return zoneId;
}
/////////////////////////////////////////////////////
/////////////// API Implementation///////////////////
/////////////////////////////////////////////////////
@Override
public String getCommandName() {
return APINAME.toLowerCase() + BaseCmd.RESPONSE_SUFFIX;
}
@Override
public long getEntityOwnerId() {
return CallContext.current().getCallingAccountId();
}
private void setupResponse(final boolean result) {
final SuccessResponse response = new SuccessResponse();
response.setSuccess(result);
response.setResponseName(getCommandName());
response.setObjectName("ha");
setResponseObject(response);
}
@Override
public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException {
final DataCenter dataCenter = _resourceService.getZone(getZoneId());
if (dataCenter == null) {
throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Unable to find zone by ID: " + getZoneId());
}
final boolean result = haConfigManager.disableHA(dataCenter);
CallContext.current().setEventDetails("Zone Id:" + dataCenter.getId() + " HA enabled: false");
CallContext.current().putContextParameter(DataCenter.class, dataCenter.getUuid());
setupResponse(result);
}
@Override
public String getEventType() {
return EventTypes.EVENT_HA_RESOURCE_DISABLE;
}
@Override
public String getEventDescription() {
return "disable HA for zone: " + getZoneId();
}
}

114
api/src/org/apache/cloudstack/api/command/admin/ha/EnableHAForClusterCmd.java Normal file
View File

@ -0,0 +1,114 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.api.command.admin.ha;
import com.cloud.event.EventTypes;
import com.cloud.exception.ConcurrentOperationException;
import com.cloud.exception.InsufficientCapacityException;
import com.cloud.exception.NetworkRuleConflictException;
import com.cloud.exception.ResourceAllocationException;
import com.cloud.exception.ResourceUnavailableException;
import com.cloud.org.Cluster;
import org.apache.cloudstack.acl.RoleType;
import org.apache.cloudstack.api.APICommand;
import org.apache.cloudstack.api.ApiArgValidator;
import org.apache.cloudstack.api.ApiConstants;
import org.apache.cloudstack.api.ApiErrorCode;
import org.apache.cloudstack.api.BaseAsyncCmd;
import org.apache.cloudstack.api.BaseCmd;
import org.apache.cloudstack.api.Parameter;
import org.apache.cloudstack.api.ServerApiException;
import org.apache.cloudstack.api.response.ClusterResponse;
import org.apache.cloudstack.api.response.SuccessResponse;
import org.apache.cloudstack.context.CallContext;
import org.apache.cloudstack.ha.HAConfigManager;
import javax.inject.Inject;
@APICommand(name = EnableHAForClusterCmd.APINAME, description = "Enables HA cluster-wide",
responseObject = SuccessResponse.class,
requestHasSensitiveInfo = false, responseHasSensitiveInfo = false,
since = "4.11", authorized = {RoleType.Admin})
public final class EnableHAForClusterCmd extends BaseAsyncCmd {
public static final String APINAME = "enableHAForCluster";
@Inject
private HAConfigManager haConfigManager;
/////////////////////////////////////////////////////
//////////////// API parameters /////////////////////
/////////////////////////////////////////////////////
@Parameter(name = ApiConstants.CLUSTER_ID, type = BaseCmd.CommandType.UUID, entityType = ClusterResponse.class,
description = "ID of the cluster", required = true, validations = {ApiArgValidator.PositiveNumber})
private Long clusterId;
/////////////////////////////////////////////////////
/////////////////// Accessors ///////////////////////
/////////////////////////////////////////////////////
public Long getClusterId() {
return clusterId;
}
/////////////////////////////////////////////////////
/////////////// API Implementation///////////////////
/////////////////////////////////////////////////////
@Override
public String getCommandName() {
return APINAME.toLowerCase() + BaseCmd.RESPONSE_SUFFIX;
}
@Override
public long getEntityOwnerId() {
return CallContext.current().getCallingAccountId();
}
private void setupResponse(final boolean result) {
final SuccessResponse response = new SuccessResponse();
response.setSuccess(result);
response.setResponseName(getCommandName());
response.setObjectName("ha");
setResponseObject(response);
}
@Override
public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException {
final Cluster cluster = _resourceService.getCluster(getClusterId());
if (cluster == null) {
throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Unable to find cluster by ID: " + getClusterId());
}
final boolean result = haConfigManager.enableHA(cluster);
CallContext.current().setEventDetails("Cluster Id:" + cluster.getId() + " HA enabled: true");
CallContext.current().putContextParameter(Cluster.class, cluster.getUuid());
setupResponse(result);
}
@Override
public String getEventType() {
return EventTypes.EVENT_HA_RESOURCE_ENABLE;
}
@Override
public String getEventDescription() {
return "enable HA for cluster: " + getClusterId();
}
}

116
api/src/org/apache/cloudstack/api/command/admin/ha/EnableHAForHostCmd.java Normal file
View File

@ -0,0 +1,116 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.api.command.admin.ha;
import com.cloud.event.EventTypes;
import com.cloud.exception.ConcurrentOperationException;
import com.cloud.exception.InsufficientCapacityException;
import com.cloud.exception.NetworkRuleConflictException;
import com.cloud.exception.ResourceAllocationException;
import com.cloud.exception.ResourceUnavailableException;
import com.cloud.host.Host;
import org.apache.cloudstack.acl.RoleType;
import org.apache.cloudstack.api.APICommand;
import org.apache.cloudstack.api.ApiArgValidator;
import org.apache.cloudstack.api.ApiConstants;
import org.apache.cloudstack.api.ApiErrorCode;
import org.apache.cloudstack.api.BaseAsyncCmd;
import org.apache.cloudstack.api.BaseCmd;
import org.apache.cloudstack.api.Parameter;
import org.apache.cloudstack.api.ServerApiException;
import org.apache.cloudstack.api.response.HostHAResponse;
import org.apache.cloudstack.api.response.HostResponse;
import org.apache.cloudstack.context.CallContext;
import org.apache.cloudstack.ha.HAConfigManager;
import org.apache.cloudstack.ha.HAResource;
import javax.inject.Inject;
@APICommand(name = EnableHAForHostCmd.APINAME, description = "Enables HA for a host",
responseObject = HostHAResponse.class,
requestHasSensitiveInfo = false, responseHasSensitiveInfo = false,
since = "4.11", authorized = {RoleType.Admin})
public final class EnableHAForHostCmd extends BaseAsyncCmd {
public static final String APINAME = "enableHAForHost";
@Inject
private HAConfigManager haConfigManager;
/////////////////////////////////////////////////////
//////////////// API parameters /////////////////////
/////////////////////////////////////////////////////
@Parameter(name = ApiConstants.HOST_ID, type = CommandType.UUID, entityType = HostResponse.class,
description = "ID of the host", required = true, validations = {ApiArgValidator.PositiveNumber})
private Long hostId;
/////////////////////////////////////////////////////
/////////////////// Accessors ///////////////////////
/////////////////////////////////////////////////////
public Long getHostId() {
return hostId;
}
/////////////////////////////////////////////////////
/////////////// API Implementation///////////////////
/////////////////////////////////////////////////////
@Override
public String getCommandName() {
return APINAME.toLowerCase() + BaseCmd.RESPONSE_SUFFIX;
}
@Override
public long getEntityOwnerId() {
return CallContext.current().getCallingAccountId();
}
private void setupResponse(final boolean result, final String resourceUuid) {
final HostHAResponse response = new HostHAResponse();
response.setId(resourceUuid);
response.setEnabled(true);
response.setStatus(result);
response.setResponseName(getCommandName());
setResponseObject(response);
}
@Override
public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException {
final Host host = _resourceService.getHost(getHostId());
if (host == null) {
throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Unable to find host by ID: " + getHostId());
}
final boolean result = haConfigManager.enableHA(host.getId(), HAResource.ResourceType.Host);
CallContext.current().setEventDetails("Host Id:" + host.getId() + " HA enabled: true");
CallContext.current().putContextParameter(Host.class, host.getUuid());
setupResponse(result, host.getUuid());
}
@Override
public String getEventType() {
return EventTypes.EVENT_HA_RESOURCE_ENABLE;
}
@Override
public String getEventDescription() {
return "enable HA for host: " + getHostId();
}
}

115
api/src/org/apache/cloudstack/api/command/admin/ha/EnableHAForZoneCmd.java Normal file
View File

@ -0,0 +1,115 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.api.command.admin.ha;
import com.cloud.dc.DataCenter;
import com.cloud.event.EventTypes;
import com.cloud.exception.ConcurrentOperationException;
import com.cloud.exception.InsufficientCapacityException;
import com.cloud.exception.NetworkRuleConflictException;
import com.cloud.exception.ResourceAllocationException;
import com.cloud.exception.ResourceUnavailableException;
import org.apache.cloudstack.acl.RoleType;
import org.apache.cloudstack.api.APICommand;
import org.apache.cloudstack.api.ApiArgValidator;
import org.apache.cloudstack.api.ApiConstants;
import org.apache.cloudstack.api.ApiErrorCode;
import org.apache.cloudstack.api.BaseAsyncCmd;
import org.apache.cloudstack.api.BaseCmd;
import org.apache.cloudstack.api.Parameter;
import org.apache.cloudstack.api.ServerApiException;
import org.apache.cloudstack.api.response.SuccessResponse;
import org.apache.cloudstack.api.response.ZoneResponse;
import org.apache.cloudstack.context.CallContext;
import org.apache.cloudstack.ha.HAConfigManager;
import javax.inject.Inject;
@APICommand(name = EnableHAForZoneCmd.APINAME, description = "Enables HA for a zone",
responseObject = SuccessResponse.class,
requestHasSensitiveInfo = false, responseHasSensitiveInfo = false,
since = "4.11", authorized = {RoleType.Admin})
public final class EnableHAForZoneCmd extends BaseAsyncCmd {
public static final String APINAME = "enableHAForZone";
@Inject
private HAConfigManager haConfigManager;
/////////////////////////////////////////////////////
//////////////// API parameters /////////////////////
/////////////////////////////////////////////////////
@Parameter(name = ApiConstants.ZONE_ID, type = BaseCmd.CommandType.UUID, entityType = ZoneResponse.class,
description = "ID of the zone", required = true, validations = {ApiArgValidator.PositiveNumber})
private Long zoneId;
/////////////////////////////////////////////////////
/////////////////// Accessors ///////////////////////
/////////////////////////////////////////////////////
public Long getZoneId() {
return zoneId;
}
/////////////////////////////////////////////////////
/////////////// API Implementation///////////////////
/////////////////////////////////////////////////////
@Override
public String getCommandName() {
return APINAME.toLowerCase() + BaseCmd.RESPONSE_SUFFIX;
}
@Override
public long getEntityOwnerId() {
return CallContext.current().getCallingAccountId();
}
private void setupResponse(final boolean result) {
final SuccessResponse response = new SuccessResponse();
response.setSuccess(result);
response.setResponseName(getCommandName());
response.setObjectName("ha");
setResponseObject(response);
}
@Override
public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException {
final DataCenter dataCenter = _resourceService.getZone(getZoneId());
if (dataCenter == null) {
throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Unable to find zone by ID: " + getZoneId());
}
final boolean result = haConfigManager.enableHA(dataCenter);
CallContext.current().setEventDetails("Zone Id:" + dataCenter.getId() + " HA enabled: true");
CallContext.current().putContextParameter(DataCenter.class, dataCenter.getUuid());
setupResponse(result);
}
@Override
public String getEventType() {
return EventTypes.EVENT_HA_RESOURCE_ENABLE;
}
@Override
public String getEventDescription() {
return "enable HA for zone: " + getZoneId();
}
}

106
api/src/org/apache/cloudstack/api/command/admin/ha/ListHostHAProvidersCmd.java Normal file
View File

@ -0,0 +1,106 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.api.command.admin.ha;
import com.cloud.exception.ConcurrentOperationException;
import com.cloud.exception.InsufficientCapacityException;
import com.cloud.exception.NetworkRuleConflictException;
import com.cloud.exception.ResourceAllocationException;
import com.cloud.exception.ResourceUnavailableException;
import com.cloud.user.Account;
import com.google.common.base.Enums;
import org.apache.cloudstack.acl.RoleType;
import org.apache.cloudstack.api.APICommand;
import org.apache.cloudstack.api.ApiArgValidator;
import org.apache.cloudstack.api.ApiConstants;
import org.apache.cloudstack.api.ApiErrorCode;
import org.apache.cloudstack.api.BaseCmd;
import org.apache.cloudstack.api.Parameter;
import org.apache.cloudstack.api.ServerApiException;
import org.apache.cloudstack.api.response.HAProviderResponse;
import org.apache.cloudstack.api.response.HostHAResponse;
import org.apache.cloudstack.api.response.ListResponse;
import org.apache.cloudstack.ha.HAConfigManager;
import org.apache.cloudstack.ha.HAResource;
import javax.inject.Inject;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@APICommand(name = ListHostHAProvidersCmd.APINAME, description = "Lists HA providers", responseObject = HostHAResponse.class,
requestHasSensitiveInfo = false, responseHasSensitiveInfo = false,
since = "4.11", authorized = {RoleType.Admin})
public final class ListHostHAProvidersCmd extends BaseCmd {
public static final String APINAME = "listHostHAProviders";
@Inject
private HAConfigManager haConfigManager;
/////////////////////////////////////////////////////
//////////////// API parameters /////////////////////
/////////////////////////////////////////////////////
@Parameter(name = ApiConstants.HYPERVISOR, type = CommandType.STRING, required = true,
description = "Hypervisor type of the resource", validations = {ApiArgValidator.NotNullOrEmpty})
private String hypervisorType;
/////////////////////////////////////////////////////
/////////////////// Accessors ///////////////////////
/////////////////////////////////////////////////////
public HAResource.ResourceSubType getHypervisorType() {
return HAResource.ResourceSubType.valueOf(hypervisorType);
}
/////////////////////////////////////////////////////
/////////////// API Implementation///////////////////
/////////////////////////////////////////////////////
@Override
public String getCommandName() {
return APINAME.toLowerCase() + BaseCmd.RESPONSE_SUFFIX;
}
@Override
public long getEntityOwnerId() {
return Account.ACCOUNT_ID_SYSTEM;
}
private void setupResponse(final List<String> hostHAProviderList) {
final ListResponse<HAProviderResponse> response = new ListResponse<>();
final List<HAProviderResponse> hostHAResponses = new ArrayList<>();
for (final String provider : hostHAProviderList) {
final HAProviderResponse haProviderResponse = new HAProviderResponse();
haProviderResponse.setProvider(provider);
hostHAResponses.add(haProviderResponse);
}
response.setResponses(hostHAResponses);
response.setResponseName(getCommandName());
setResponseObject(response);
}
@Override
public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException {
if (!Enums.getIfPresent(HAResource.ResourceSubType.class, hypervisorType).isPresent()) {
throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Invalid or unsupported host hypervisor type provided. Supported types are: " + Arrays.toString(HAResource.ResourceSubType.values()));
}
final List<String> hostHAProviders = haConfigManager.listHAProviders(HAResource.ResourceType.Host, getHypervisorType());
setupResponse(hostHAProviders);
}
}

109
api/src/org/apache/cloudstack/api/command/admin/ha/ListHostHAResourcesCmd.java Normal file
View File

@ -0,0 +1,109 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.api.command.admin.ha;
import com.cloud.exception.ConcurrentOperationException;
import com.cloud.exception.InsufficientCapacityException;
import com.cloud.exception.NetworkRuleConflictException;
import com.cloud.exception.ResourceAllocationException;
import com.cloud.exception.ResourceUnavailableException;
import com.cloud.host.Host;
import com.cloud.user.Account;
import org.apache.cloudstack.acl.RoleType;
import org.apache.cloudstack.api.APICommand;
import org.apache.cloudstack.api.ApiArgValidator;
import org.apache.cloudstack.api.ApiConstants;
import org.apache.cloudstack.api.BaseCmd;
import org.apache.cloudstack.api.Parameter;
import org.apache.cloudstack.api.ServerApiException;
import org.apache.cloudstack.api.response.HostHAResponse;
import org.apache.cloudstack.api.response.HostResponse;
import org.apache.cloudstack.api.response.ListResponse;
import org.apache.cloudstack.ha.HAConfig;
import org.apache.cloudstack.ha.HAConfigManager;
import org.apache.cloudstack.ha.HAResource;
import javax.inject.Inject;
import java.util.ArrayList;
import java.util.List;
@APICommand(name = ListHostHAResourcesCmd.APINAME, description = "Lists host HA resources", responseObject = HostHAResponse.class,
requestHasSensitiveInfo = false, responseHasSensitiveInfo = false,
since = "4.11", authorized = {RoleType.Admin})
public final class ListHostHAResourcesCmd extends BaseCmd {
public static final String APINAME = "listHostHAResources";
@Inject
private HAConfigManager haConfigManager;
/////////////////////////////////////////////////////
//////////////// API parameters /////////////////////
/////////////////////////////////////////////////////
@Parameter(name = ApiConstants.HOST_ID, type = CommandType.UUID, entityType = HostResponse.class,
description = "List by host ID", validations = {ApiArgValidator.PositiveNumber})
private Long hostId;
/////////////////////////////////////////////////////
/////////////////// Accessors ///////////////////////
/////////////////////////////////////////////////////
public Long getHostId() {
return hostId;
}
/////////////////////////////////////////////////////
/////////////// API Implementation///////////////////
/////////////////////////////////////////////////////
@Override
public String getCommandName() {
return APINAME.toLowerCase() + BaseCmd.RESPONSE_SUFFIX;
}
@Override
public long getEntityOwnerId() {
return Account.ACCOUNT_ID_SYSTEM;
}
private void setupResponse(final List<HAConfig> hostHAConfigList) {
final ListResponse<HostHAResponse> response = new ListResponse<>();
final List<HostHAResponse> hostHAResponses = new ArrayList<>();
for (final HAConfig config : hostHAConfigList) {
final Host host = _resourceService.getHost(config.getResourceId());
if (host == null) {
continue;
}
final HostHAResponse hostHAResponse = new HostHAResponse();
hostHAResponse.setId(host.getUuid());
hostHAResponse.setEnabled(config.isEnabled());
hostHAResponse.setHaState(config.getState());
hostHAResponse.setProvider(config.getHaProvider());
hostHAResponses.add(hostHAResponse);
}
response.setResponses(hostHAResponses);
response.setResponseName(getCommandName());
setResponseObject(response);
}
@Override
public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException {
final List<HAConfig> hostHAConfig = haConfigManager.listHAResources(getHostId(), HAResource.ResourceType.Host);
setupResponse(hostHAConfig);
}
}

4
api/src/org/apache/cloudstack/api/command/admin/host/PrepareForMaintenanceCmd.java
View File

@ -108,4 +108,8 @@ public class PrepareForMaintenanceCmd extends BaseAsyncCmd {
throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to prepare host for maintenance");
}
}
public void setHostId(final Long hostId) {
id = hostId;
}
}

2
api/src/org/apache/cloudstack/api/command/admin/outofbandmanagement/ChangeOutOfBandManagementPasswordCmd.java
View File

@ -74,7 +74,7 @@ public class ChangeOutOfBandManagementPasswordCmd extends BaseAsyncCmd {
CallContext.current().setEventDetails("Host Id: " + host.getId() + " Password: " + getPassword().charAt(0) + "****");
CallContext.current().putContextParameter(Host.class, host.getUuid());
final OutOfBandManagementResponse response = outOfBandManagementService.changeOutOfBandManagementPassword(host, getPassword());
final OutOfBandManagementResponse response = outOfBandManagementService.changePassword(host, getPassword());
response.setResponseName(getCommandName());
setResponseObject(response);
}

2
api/src/org/apache/cloudstack/api/command/admin/outofbandmanagement/ConfigureOutOfBandManagementCmd.java
View File

@ -83,7 +83,7 @@ public class ConfigureOutOfBandManagementCmd extends BaseCmd {
throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Unable to find host by ID: " + getHostId());
}
CallContext.current().putContextParameter(Host.class, host.getUuid());
final OutOfBandManagementResponse response = outOfBandManagementService.configureOutOfBandManagement(host, getHostPMOptions());
final OutOfBandManagementResponse response = outOfBandManagementService.configure(host, getHostPMOptions());
response.setId(host.getUuid());
response.setResponseName(getCommandName());
setResponseObject(response);

2
api/src/org/apache/cloudstack/api/command/admin/outofbandmanagement/IssueOutOfBandManagementPowerActionCmd.java
View File

@ -80,7 +80,7 @@ public class IssueOutOfBandManagementPowerActionCmd extends BaseAsyncCmd {
CallContext.current().setEventDetails("Host Id: " + host.getId() + " Action: " + powerOperation.toString());
CallContext.current().putContextParameter(Host.class, host.getUuid());
final OutOfBandManagementResponse response = outOfBandManagementService.executeOutOfBandManagementPowerOperation(host, powerOperation, getActionTimeout());
final OutOfBandManagementResponse response = outOfBandManagementService.executePowerOperation(host, powerOperation, getActionTimeout());
response.setResponseName(getCommandName());
setResponseObject(response);
}

58
api/src/org/apache/cloudstack/api/response/HAProviderResponse.java Normal file
View File

@ -0,0 +1,58 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.api.response;
import com.cloud.serializer.Param;
import com.google.gson.annotations.SerializedName;
import org.apache.cloudstack.api.ApiConstants;
import org.apache.cloudstack.api.BaseResponse;
import org.apache.cloudstack.api.EntityReference;
import org.apache.cloudstack.ha.HAConfig;
import java.util.List;
@EntityReference(value = HAConfig.class)
public final class HAProviderResponse extends BaseResponse {
@SerializedName(ApiConstants.HA_PROVIDER)
@Param(description = "the HA provider")
private String provider;
@SerializedName(ApiConstants.TYPE)
@Param(description = "the HA provider resource type detail")
private List<String> supportedResourceTypes;
public HAProviderResponse() {
super("haprovider");
}
public String getProvider() {
return provider;
}
public void setProvider(String provider) {
this.provider = provider;
}
public List<String> getSupportedResourceTypes() {
return supportedResourceTypes;
}
public void setSupportedResourceTypes(List<String> supportedResourceTypes) {
this.supportedResourceTypes = supportedResourceTypes;
}
}

104
api/src/org/apache/cloudstack/api/response/HostHAResponse.java Normal file
View File

@ -0,0 +1,104 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.api.response;
import com.cloud.serializer.Param;
import com.google.gson.annotations.SerializedName;
import org.apache.cloudstack.api.ApiConstants;
import org.apache.cloudstack.api.BaseResponse;
import org.apache.cloudstack.api.EntityReference;
import org.apache.cloudstack.ha.HAConfig;
@EntityReference(value = HAConfig.class)
public final class HostHAResponse extends BaseResponse {
@SerializedName(ApiConstants.HOST_ID)
@Param(description = "the ID of the host")
private String id;
@SerializedName(ApiConstants.HA_ENABLE)
@Param(description = "if host HA is enabled for the host")
private Boolean enabled;
@SerializedName(ApiConstants.HA_STATE)
@Param(description = "the HA state of the host")
private HAConfig.HAState haState;
@SerializedName(ApiConstants.HA_PROVIDER)
@Param(description = "the host HA provider")
private String provider;
@SerializedName(ApiConstants.STATUS)
@Param(description = "operation status")
private Boolean status;
public HostHAResponse() {
super("hostha");
}
public HostHAResponse(final HAConfig config) {
this();
if (config == null) {
this.enabled = false;
this.haState = HAConfig.HAState.Disabled;
return;
}
setProvider(config.getHaProvider());
setEnabled(config.isEnabled());
setHaState(config.getState());
}
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
public Boolean getEnabled() {
return enabled;
}
public void setEnabled(Boolean enabled) {
this.enabled = enabled;
}
public HAConfig.HAState getHaState() {
return haState;
}
public void setHaState(HAConfig.HAState haState) {
this.haState = haState;
}
public String getProvider() {
return provider;
}
public void setProvider(String provider) {
this.provider = provider;
}
public Boolean getStatus() {
return status;
}
public void setStatus(Boolean status) {
this.status = status;
}
}

13
api/src/org/apache/cloudstack/api/response/HostResponse.java
View File

@ -24,6 +24,7 @@ import com.google.gson.annotations.SerializedName;
import org.apache.cloudstack.api.ApiConstants;
import org.apache.cloudstack.api.BaseResponse;
import org.apache.cloudstack.api.EntityReference;
import org.apache.cloudstack.ha.HAConfig;
import org.apache.cloudstack.outofbandmanagement.OutOfBandManagement;
import java.util.Date;
@ -201,6 +202,10 @@ public class HostResponse extends BaseResponse {
@Param(description = "true if this host is suitable(has enough capacity and satisfies all conditions like hosttags, max guests vm limit etc) to migrate a VM to it , false otherwise")
private Boolean suitableForMigration;
@SerializedName("hostha")
@Param(description = "the host HA information information")
private HostHAResponse hostHAResponse;
@SerializedName("outofbandmanagement")
@Param(description = "the host out-of-band management information")
private OutOfBandManagementResponse outOfBandManagementResponse;
@ -408,6 +413,14 @@ public class HostResponse extends BaseResponse {
this.suitableForMigration = suitableForMigration;
}
public HostHAResponse getHostHAResponse() {
return hostHAResponse;
}
public void setHostHAResponse(final HAConfig config) {
this.hostHAResponse = new HostHAResponse(config);
}
public OutOfBandManagementResponse getOutOfBandManagementResponse() {
return outOfBandManagementResponse;
}

142
api/src/org/apache/cloudstack/ha/HAConfig.java Normal file
View File

@ -0,0 +1,142 @@
//
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//
package org.apache.cloudstack.ha;
import com.cloud.utils.fsm.StateMachine2;
import com.cloud.utils.fsm.StateObject;
import org.apache.cloudstack.api.InternalIdentity;
import org.apache.cloudstack.utils.identity.ManagementServerNode;
public interface HAConfig extends StateObject<HAConfig.HAState>, InternalIdentity {
long getResourceId();
HAResource.ResourceType getResourceType();
boolean isEnabled();
HAState getState();
String getHaProvider();
Long getManagementServerId();
enum Event {
Eligible,
Ineligible,
Disabled,
Enabled,
HealthCheckPassed,
HealthCheckFailed,
PerformActivityCheck,
TooFewActivityCheckSamples,
PeriodicRecheckResourceActivity,
ActivityCheckFailureOverThresholdRatio,
ActivityCheckFailureUnderThresholdRatio,
PowerCycle,
Recovered,
RecoveryWaitPeriodTimeout,
RecoveryOperationThresholdExceeded,
Fenced;
public Long getServerId() {
// TODO: change in future if we've better claim & ownership
// Right now the first one to update the db wins
// and mgmt server id would eventually become consistent
return ManagementServerNode.getManagementServerId();
}
}
enum HAState {
Disabled("HA Operations disabled"),
Available("The resource is healthy"),
Ineligible("The current state does not support HA/recovery"),
Suspect("Most recent health check failed"),
Degraded("The resource cannot be managed, but services end user requests"),
Checking("The activity checks are currently being performed"),
Recovering("The resource is undergoing recovery operation"),
Recovered("The resource is recovered"),
Fencing("The resource is undergoing fence operation"),
Fenced("The resource is fenced");
String description;
HAState(String description) {
this.description = description;
}
public static StateMachine2<HAState, Event, HAConfig> getStateMachine() {
return FSM;
}
public String getDescription() {
return description;
}
private static final StateMachine2<HAState, Event, HAConfig> FSM = new StateMachine2<>();
static {
FSM.addInitialTransition(Event.Disabled, Disabled);
FSM.addInitialTransition(Event.Enabled, Available);
FSM.addInitialTransition(Event.Ineligible, Ineligible);
FSM.addTransition(Disabled, Event.Enabled, Available);
FSM.addTransition(Ineligible, Event.Disabled, Disabled);
FSM.addTransition(Ineligible, Event.Ineligible, Ineligible);
FSM.addTransition(Ineligible, Event.Eligible, Available);
FSM.addTransition(Available, Event.Disabled, Disabled);
FSM.addTransition(Available, Event.Ineligible, Ineligible);
FSM.addTransition(Available, Event.HealthCheckPassed, Available);
FSM.addTransition(Available, Event.HealthCheckFailed, Suspect);
FSM.addTransition(Suspect, Event.Disabled, Disabled);
FSM.addTransition(Suspect, Event.Ineligible, Ineligible);
FSM.addTransition(Suspect, Event.HealthCheckFailed, Suspect);
FSM.addTransition(Suspect, Event.PerformActivityCheck, Checking);
FSM.addTransition(Suspect, Event.HealthCheckPassed, Available);
FSM.addTransition(Checking, Event.Disabled, Disabled);
FSM.addTransition(Checking, Event.Ineligible, Ineligible);
FSM.addTransition(Checking, Event.TooFewActivityCheckSamples, Suspect);
FSM.addTransition(Checking, Event.ActivityCheckFailureUnderThresholdRatio, Degraded);
FSM.addTransition(Checking, Event.ActivityCheckFailureOverThresholdRatio, Recovering);
FSM.addTransition(Degraded, Event.Disabled, Disabled);
FSM.addTransition(Degraded, Event.Ineligible, Ineligible);
FSM.addTransition(Degraded, Event.HealthCheckFailed, Degraded);
FSM.addTransition(Degraded, Event.HealthCheckPassed, Available);
FSM.addTransition(Degraded, Event.PeriodicRecheckResourceActivity, Suspect);
FSM.addTransition(Recovering, Event.Disabled, Disabled);
FSM.addTransition(Recovering, Event.Ineligible, Ineligible);
FSM.addTransition(Recovering, Event.Recovered, Recovered);
FSM.addTransition(Recovering, Event.RecoveryOperationThresholdExceeded, Fencing);
FSM.addTransition(Recovered, Event.Disabled, Disabled);
FSM.addTransition(Recovered, Event.Ineligible, Ineligible);
FSM.addTransition(Recovered, Event.RecoveryWaitPeriodTimeout, Available);
FSM.addTransition(Fencing, Event.Disabled, Disabled);
FSM.addTransition(Fencing, Event.Ineligible, Ineligible);
FSM.addTransition(Fencing, Event.Fenced, Fenced);
FSM.addTransition(Fenced, Event.Disabled, Disabled);
FSM.addTransition(Fenced, Event.HealthCheckPassed, Ineligible);
FSM.addTransition(Fenced, Event.HealthCheckFailed, Fenced);
}
}
}

96
api/src/org/apache/cloudstack/ha/HAConfigManager.java Normal file
View File

@ -0,0 +1,96 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.ha;
import com.cloud.dc.DataCenter;
import com.cloud.org.Cluster;
import java.util.List;
/**
* @since 4.11
*/
public interface HAConfigManager {
/**
* Configures HA for a resource by accepting the resource type and HA provider
* @param resourceId the ID of the resource
* @param resourceType the type of the resource
* @param haProvider the name of the HA provider
*/
boolean configureHA(Long resourceId, HAResource.ResourceType resourceType, String haProvider);
/**
* Enables HA for resource Id of a specific resource type
* @param resourceId the ID of the resource
* @param resourceType the type of the resource
* @return returns true on successful enable
*/
boolean enableHA(Long resourceId, HAResource.ResourceType resourceType);
/**
* Disables HA for resource Id of a specific resource type
* @param resourceId the ID of the resource
* @param resourceType the type of the resource
* @return returns true on successful disable
*/
boolean disableHA(Long resourceId, HAResource.ResourceType resourceType);
/**
* Enables HA across a cluster
* @param cluster the cluster
* @return returns operation success
*/
boolean enableHA(final Cluster cluster);
/**
* Disables HA across a cluster
* @param cluster the cluster
* @return returns operation success
*/
boolean disableHA(final Cluster cluster);
/**
* Enables HA across a zone
* @param zone the zone
* @return returns operation success
*/
boolean enableHA(final DataCenter zone);
/**
* Disables HA across a zone
* @param zone the zone
* @return returns operation success
*/
boolean disableHA(final DataCenter zone);
/**
* Returns list of HA config for resources, by resource ID and/or type if provided
* @param resourceId (optional) ID of the resource
* @param resourceType (optional) type of the resource
* @return returns list of ha config for the resource
*/
List<HAConfig> listHAResources(final Long resourceId, final HAResource.ResourceType resourceType);
/**
* Returns list of HA providers for resources
* @param resourceType type of the resource
* @param entityType sub-type of the resource
* @return returns list of ha provider names
*/
List<String> listHAProviders(final HAResource.ResourceType resourceType, final HAResource.ResourceSubType entityType);
}

37
api/src/org/apache/cloudstack/ha/HAResource.java Normal file
View File

@ -0,0 +1,37 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.ha;
import org.apache.cloudstack.kernel.Partition;
public interface HAResource extends Partition {
enum ResourceType {
Host,
VirtualMachine
}
enum ResourceSubType {
KVM,
Simulator,
Unknown
}
long getDataCenterId();
Long getClusterId();
ResourceType resourceType();
}

28
api/src/org/apache/cloudstack/kernel/Partition.java Normal file
View File

@ -0,0 +1,28 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.kernel;
import org.apache.cloudstack.api.Identity;
import org.apache.cloudstack.api.InternalIdentity;
public interface Partition extends InternalIdentity, Identity {
enum PartitionType {
Zone, Pod, Cluster, Host, VM
}
PartitionType partitionType();
}

9
api/src/org/apache/cloudstack/outofbandmanagement/OutOfBandManagementService.java
View File

@ -30,9 +30,6 @@ public interface OutOfBandManagementService {
ConfigKey<Long> ActionTimeout = new ConfigKey<Long>("Advanced", Long.class, "outofbandmanagement.action.timeout", "60",
"The out of band management action timeout in seconds, configurable by cluster", true, ConfigKey.Scope.Cluster);
ConfigKey<Long> SyncThreadInterval = new ConfigKey<Long>("Advanced", Long.class, "outofbandmanagement.sync.interval", "300000",
"The interval (in milliseconds) when the out-of-band management background sync are retrieved", true, ConfigKey.Scope.Global);
ConfigKey<Integer> SyncThreadPoolSize = new ConfigKey<Integer>("Advanced", Integer.class, "outofbandmanagement.sync.poolsize", "50",
"The out of band management background sync thread pool size", true, ConfigKey.Scope.Global);
@ -49,7 +46,7 @@ public interface OutOfBandManagementService {
OutOfBandManagementResponse disableOutOfBandManagement(Cluster cluster);
OutOfBandManagementResponse disableOutOfBandManagement(Host host);
OutOfBandManagementResponse configureOutOfBandManagement(Host host, ImmutableMap<OutOfBandManagement.Option, String> options);
OutOfBandManagementResponse executeOutOfBandManagementPowerOperation(Host host, OutOfBandManagement.PowerOperation operation, Long timeout);
OutOfBandManagementResponse changeOutOfBandManagementPassword(Host host, String password);
OutOfBandManagementResponse configure(Host host, ImmutableMap<OutOfBandManagement.Option, String> options);
OutOfBandManagementResponse executePowerOperation(Host host, OutOfBandManagement.PowerOperation operation, Long timeout);
OutOfBandManagementResponse changePassword(Host host, String password);
}

30
api/src/org/apache/cloudstack/poll/BackgroundPollManager.java Normal file
View File

@ -0,0 +1,30 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.poll;
public interface BackgroundPollManager {
/**
* Submits a background poll task that need to run continuously in the background
* to poll external resources, update states, trigger actions etc.
* Tasks must be submitted by a manager in configure-phase, the list of submitted tasks
* are then submitted to the internal executor service during start-phase.
* @param task periodic background task
* @since 4.11
*/
void submitTask(final BackgroundPollTask task);
}

21
api/src/org/apache/cloudstack/poll/BackgroundPollTask.java Normal file
View File

@ -0,0 +1,21 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.poll;
public interface BackgroundPollTask extends Runnable {
}

7
client/WEB-INF/classes/resources/messages.properties
View File

@ -639,6 +639,12 @@ label.guest.traffic=Guest Traffic
label.guest.type=Guest Type
label.guest=Guest
label.ha.enabled=HA Enabled
label.ha.configure=Configure HA
label.ha.disable=Disable HA
label.ha.enable=Enable HA
label.ha.provider=HA Provider
label.ha.state=HA State
label.ha=HA
label.help=Help
label.hide.ingress.rule=Hide Ingress Rule
label.hints=Hints
@ -998,6 +1004,7 @@ label.outofbandmanagement.driver=Driver
label.outofbandmanagement.disable=Disable Out-of-band Management
label.outofbandmanagement.enable=Enable Out-of-band Management
label.outofbandmanagement.password=Password
label.outofbandmanagement.reenterpassword=Re-enter Password
label.outofbandmanagement.port=Port
label.outofbandmanagement.username=Username
message.outofbandmanagement.changepassword=Change Out-of-band Management password

5
core/resources/META-INF/cloudstack/compute/spring-core-lifecycle-compute-context-inheritable.xml
View File

@ -44,4 +44,9 @@
<property name="typeClass" value="com.cloud.hypervisor.HypervisorGuru" />
</bean>
<bean class="org.apache.cloudstack.spring.lifecycle.registry.RegistryLifecycle">
<property name="registry" ref="haProvidersRegistry" />
<property name="typeClass" value="org.apache.cloudstack.ha.provider.HAProvider" />
</bean>
</beans>

5
core/resources/META-INF/cloudstack/core/spring-core-registry-core-context.xml
View File

@ -307,6 +307,11 @@
<property name="excludeKey" value="data.motion.strategies.exclude" />
</bean>
<bean id="haProvidersRegistry"
class="org.apache.cloudstack.spring.lifecycle.registry.ExtensionRegistry">
<property name="orderConfigDefault" value="KVMHAProvider" />
</bean>
<bean id="outOfBandManagementDriversRegistry"
class="org.apache.cloudstack.spring.lifecycle.registry.ExtensionRegistry">
<property name="orderConfigDefault" value="IPMITOOL" />

70
core/src/com/cloud/agent/api/CheckVMActivityOnStoragePoolCommand.java Normal file
View File

@ -0,0 +1,70 @@
//
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//
package com.cloud.agent.api;
import com.cloud.agent.api.to.HostTO;
import com.cloud.agent.api.to.StorageFilerTO;
import com.cloud.host.Host;
import com.cloud.storage.StoragePool;
import com.cloud.storage.Volume;
import org.joda.time.DateTime;
import java.util.List;
public final class CheckVMActivityOnStoragePoolCommand extends Command {
private HostTO host;
private StorageFilerTO pool;
private String volumeList;
private long suspectTimeSeconds;
public CheckVMActivityOnStoragePoolCommand(final Host host, final StoragePool pool, final List<Volume> volumeList, final DateTime suspectTime) {
this.host = new HostTO(host);
this.pool = new StorageFilerTO(pool);
this.suspectTimeSeconds = suspectTime.getMillis()/1000L;
final StringBuilder stringBuilder = new StringBuilder();
for (final Volume v : volumeList) {
stringBuilder.append(v.getUuid()).append(",");
}
this.volumeList = stringBuilder.deleteCharAt(stringBuilder.length() - 1).toString();
}
public String getVolumeList() {
return volumeList;
}
public StorageFilerTO getPool() {
return pool;
}
public HostTO getHost() {
return host;
}
public long getSuspectTimeInSeconds() {
return suspectTimeSeconds;
}
@Override
public boolean executeInSequence() {
return false;
}
}

19
core/test/org/apache/cloudstack/api/agent/test/CheckOnHostCommandTest.java
View File

@ -27,6 +27,8 @@ import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
import org.apache.cloudstack.ha.HAResource;
import org.apache.cloudstack.kernel.Partition;
import org.junit.Test;
import com.cloud.agent.api.CheckOnHostCommand;
@ -38,6 +40,11 @@ import com.cloud.resource.ResourceState;
public class CheckOnHostCommandTest {
public Host host = new Host() {
@Override
public PartitionType partitionType() {
return PartitionType.Host;
}
@Override
public Status getState() {
return Status.Up;
@ -197,7 +204,12 @@ public class CheckOnHostCommandTest {
@Override
public Long getClusterId() {
return 3L;
};
}
@Override
public ResourceType resourceType() {
return ResourceType.Host;
}
@Override
public String getPublicIpAddress() {
@ -254,6 +266,11 @@ public class CheckOnHostCommandTest {
return false;
};
@Override
public boolean isDisabled() {
return false;
};
@Override
public ResourceState getResourceState() {
return ResourceState.Enabled;

4
developer/developer-prefill.sql
View File

@ -78,10 +78,6 @@ INSERT INTO `cloud`.`configuration` (category, instance, component, name, value)
VALUES ('Advanced', 'DEFAULT', 'management-server',
'pool.storage.capacity.disablethreshold', '0.95');
INSERT INTO `cloud`.`configuration` (category, instance, component, name, value)
VALUES ('Advanced', 'DEFAULT', 'management-server',
'outofbandmanagement.sync.interval', '2000');
-- Enable dynamic RBAC by default for fresh deployments
INSERT INTO `cloud`.`configuration` (category, instance, component, name, value)
VALUES ('Advanced', 'DEFAULT', 'RoleService',

2
engine/components-api/src/com/cloud/agent/AgentManager.java
View File

@ -139,6 +139,8 @@ public interface AgentManager {
void disconnectWithoutInvestigation(long hostId, Status.Event event);
void disconnectWithInvestigation(long hostId, Status.Event event);
public void pullAgentToMaintenance(long hostId);
public void pullAgentOutMaintenance(long hostId);

2
engine/orchestration/src/com/cloud/agent/manager/AgentManagerImpl.java
View File

@ -1201,7 +1201,7 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
} else if (cmd instanceof ShutdownCommand) {
final ShutdownCommand shutdown = (ShutdownCommand)cmd;
final String reason = shutdown.getReason();
s_logger.info("Host " + attache.getId() + " has informed us that it is shutting down with reason " + reason + " and detail " +
s_logger.info("HA: Host " + attache.getId() + " has informed us that it is shutting down with reason " + reason + " and detail " +
shutdown.getDetail());
if (reason.equals(ShutdownCommand.Update)) {
//disconnectWithoutInvestigation(attache, Event.UpdateNeeded);

9
engine/orchestration/src/com/cloud/agent/manager/ClusteredAgentManagerImpl.java
View File

@ -43,6 +43,8 @@ import javax.naming.ConfigurationException;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLEngine;
import org.apache.cloudstack.ha.dao.HAConfigDao;
import org.apache.cloudstack.outofbandmanagement.dao.OutOfBandManagementDao;
import org.apache.log4j.Logger;
import com.google.gson.Gson;
@ -124,6 +126,10 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
ConfigurationDao _configDao;
@Inject
ConfigDepot _configDepot;
@Inject
private OutOfBandManagementDao outOfBandManagementDao;
@Inject
private HAConfigDao haConfigDao;
protected ClusteredAgentManagerImpl() {
super();
@ -729,7 +735,8 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
s_logger.info("Marking hosts as disconnected on Management server" + vo.getMsid());
long lastPing = (System.currentTimeMillis() >> 10) - getTimeout();
_hostDao.markHostsAsDisconnected(vo.getMsid(), lastPing);
outOfBandManagementDao.expireOutOfBandManagementOwnershipByServer(vo.getMsid());
outOfBandManagementDao.expireServerOwnership(vo.getMsid());
haConfigDao.expireServerOwnership(vo.getMsid());
s_logger.info("Deleting entries from op_host_transfer table for Management server " + vo.getMsid());
cleanupTransferMap(vo.getMsid());
}

31
engine/orchestration/src/org/apache/cloudstack/engine/datacenter/entity/api/db/EngineClusterVO.java
View File

@ -16,8 +16,16 @@
// under the License.
package org.apache.cloudstack.engine.datacenter.entity.api.db;
import java.util.Date;
import java.util.UUID;
import com.cloud.hypervisor.Hypervisor.HypervisorType;
import com.cloud.org.Cluster;
import com.cloud.org.Grouping;
import com.cloud.org.Managed.ManagedState;
import com.cloud.utils.NumbersUtil;
import com.cloud.utils.db.GenericDao;
import com.cloud.utils.db.StateMachine;
import org.apache.cloudstack.api.Identity;
import org.apache.cloudstack.engine.datacenter.entity.api.DataCenterResourceEntity.State;
import org.apache.cloudstack.engine.datacenter.entity.api.DataCenterResourceEntity.State.Event;
import javax.persistence.Column;
import javax.persistence.Entity;
@ -29,18 +37,8 @@ import javax.persistence.Id;
import javax.persistence.Table;
import javax.persistence.Temporal;
import javax.persistence.TemporalType;
import org.apache.cloudstack.api.Identity;
import org.apache.cloudstack.engine.datacenter.entity.api.DataCenterResourceEntity.State;
import org.apache.cloudstack.engine.datacenter.entity.api.DataCenterResourceEntity.State.Event;
import com.cloud.hypervisor.Hypervisor.HypervisorType;
import com.cloud.org.Cluster;
import com.cloud.org.Grouping;
import com.cloud.org.Managed.ManagedState;
import com.cloud.utils.NumbersUtil;
import com.cloud.utils.db.GenericDao;
import com.cloud.utils.db.StateMachine;
import java.util.Date;
import java.util.UUID;
@Entity
@Table(name = "cluster")
@ -243,4 +241,9 @@ public class EngineClusterVO implements EngineCluster, Identity {
public State getState() {
return state;
}
@Override
public PartitionType partitionType() {
return PartitionType.Cluster;
}
}

29
engine/orchestration/src/org/apache/cloudstack/engine/datacenter/entity/api/db/EngineDataCenterVO.java
View File

@ -16,9 +16,14 @@
// under the License.
package org.apache.cloudstack.engine.datacenter.entity.api.db;
import java.util.Date;
import java.util.Map;
import java.util.UUID;
import com.cloud.network.Network.Provider;
import com.cloud.org.Grouping;
import com.cloud.utils.NumbersUtil;
import com.cloud.utils.db.GenericDao;
import com.cloud.utils.db.StateMachine;
import org.apache.cloudstack.api.Identity;
import org.apache.cloudstack.engine.datacenter.entity.api.DataCenterResourceEntity.State;
import org.apache.cloudstack.engine.datacenter.entity.api.DataCenterResourceEntity.State.Event;
import javax.persistence.Column;
import javax.persistence.Entity;
@ -32,16 +37,9 @@ import javax.persistence.TableGenerator;
import javax.persistence.Temporal;
import javax.persistence.TemporalType;
import javax.persistence.Transient;
import org.apache.cloudstack.api.Identity;
import org.apache.cloudstack.engine.datacenter.entity.api.DataCenterResourceEntity.State;
import org.apache.cloudstack.engine.datacenter.entity.api.DataCenterResourceEntity.State.Event;
import com.cloud.network.Network.Provider;
import com.cloud.org.Grouping;
import com.cloud.utils.NumbersUtil;
import com.cloud.utils.db.GenericDao;
import com.cloud.utils.db.StateMachine;
import java.util.Date;
import java.util.Map;
import java.util.UUID;
@Entity
@Table(name = "data_center")
@ -503,4 +501,9 @@ public class EngineDataCenterVO implements EngineDataCenter, Identity {
public void setIp6Dns2(String ip6Dns2) {
this.ip6Dns2 = ip6Dns2;
}
@Override
public PartitionType partitionType() {
return PartitionType.Zone;
}
}

45
engine/orchestration/src/org/apache/cloudstack/engine/datacenter/entity/api/db/EngineHostVO.java
View File

@ -16,10 +16,16 @@
// under the License.
package org.apache.cloudstack.engine.datacenter.entity.api.db;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import com.cloud.host.Status;
import com.cloud.hypervisor.Hypervisor.HypervisorType;
import com.cloud.resource.ResourceState;
import com.cloud.storage.Storage.StoragePoolType;
import com.cloud.utils.NumbersUtil;
import com.cloud.utils.db.GenericDao;
import com.cloud.utils.db.StateMachine;
import org.apache.cloudstack.api.Identity;
import org.apache.cloudstack.engine.datacenter.entity.api.DataCenterResourceEntity.State;
import org.apache.cloudstack.engine.datacenter.entity.api.DataCenterResourceEntity.State.Event;
import javax.persistence.Column;
import javax.persistence.DiscriminatorColumn;
@ -36,18 +42,10 @@ import javax.persistence.Table;
import javax.persistence.Temporal;
import javax.persistence.TemporalType;
import javax.persistence.Transient;
import org.apache.cloudstack.api.Identity;
import org.apache.cloudstack.engine.datacenter.entity.api.DataCenterResourceEntity.State;
import org.apache.cloudstack.engine.datacenter.entity.api.DataCenterResourceEntity.State.Event;
import com.cloud.host.Status;
import com.cloud.hypervisor.Hypervisor.HypervisorType;
import com.cloud.resource.ResourceState;
import com.cloud.storage.Storage.StoragePoolType;
import com.cloud.utils.NumbersUtil;
import com.cloud.utils.db.GenericDao;
import com.cloud.utils.db.StateMachine;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.UUID;
@Entity
@Table(name = "host")
@ -178,6 +176,11 @@ public class EngineHostVO implements EngineHost, Identity {
return clusterId;
}
@Override
public ResourceType resourceType() {
return ResourceType.Host;
}
public void setClusterId(Long clusterId) {
this.clusterId = clusterId;
}
@ -722,6 +725,11 @@ public class EngineHostVO implements EngineHost, Identity {
resourceState = state;
}
@Override
public boolean isDisabled() {
return (getResourceState() == ResourceState.Disabled);
}
@Override
public boolean isInMaintenanceStates() {
return (getResourceState() == ResourceState.Maintenance || getResourceState() == ResourceState.ErrorInMaintenance || getResourceState() == ResourceState.PrepareForMaintenance);
@ -760,4 +768,9 @@ public class EngineHostVO implements EngineHost, Identity {
public State getOrchestrationState() {
return orchestrationState;
}
@Override
public PartitionType partitionType() {
return PartitionType.Host;
}
}

2
engine/schema/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml
View File

@ -165,6 +165,8 @@
<bean id="guestOSCategoryDaoImpl" class="com.cloud.storage.dao.GuestOSCategoryDaoImpl" />
<bean id="guestOSDaoImpl" class="com.cloud.storage.dao.GuestOSDaoImpl" />
<bean id="guestOSHypervisorDaoImpl" class="com.cloud.storage.dao.GuestOSHypervisorDaoImpl" />
<!-- New HA Config -->
<bean id="haConfigDaoImpl" class="org.apache.cloudstack.ha.dao.HAConfigDaoImpl" />
<bean id="highAvailabilityDaoImpl" class="com.cloud.ha.dao.HighAvailabilityDaoImpl" />
<bean id="hostDaoImpl" class="com.cloud.host.dao.HostDaoImpl" />
<bean id="engineHostDetailsDaoImpl" class="org.apache.cloudstack.engine.datacenter.entity.api.db.dao.HostDetailsDaoImpl" />

22
engine/schema/src/com/cloud/dc/ClusterVO.java
View File

@ -16,8 +16,12 @@
// under the License.
package com.cloud.dc;
import java.util.Date;
import java.util.UUID;
import com.cloud.hypervisor.Hypervisor.HypervisorType;
import com.cloud.org.Cluster;
import com.cloud.org.Grouping;
import com.cloud.org.Managed.ManagedState;
import com.cloud.utils.NumbersUtil;
import com.cloud.utils.db.GenericDao;
import javax.persistence.Column;
import javax.persistence.Entity;
@ -27,13 +31,8 @@ import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.Table;
import com.cloud.hypervisor.Hypervisor.HypervisorType;
import com.cloud.org.Cluster;
import com.cloud.org.Grouping;
import com.cloud.org.Managed.ManagedState;
import com.cloud.utils.NumbersUtil;
import com.cloud.utils.db.GenericDao;
import java.util.Date;
import java.util.UUID;
@Entity
@Table(name = "cluster")
@ -192,4 +191,9 @@ public class ClusterVO implements Cluster {
public void setUuid(String uuid) {
this.uuid = uuid;
}
@Override
public PartitionType partitionType() {
return PartitionType.Cluster;
}
}

20
engine/schema/src/com/cloud/dc/DataCenterVO.java
View File

@ -16,9 +16,10 @@
// under the License.
package com.cloud.dc;
import java.util.Date;
import java.util.Map;
import java.util.UUID;
import com.cloud.network.Network.Provider;
import com.cloud.org.Grouping;
import com.cloud.utils.NumbersUtil;
import com.cloud.utils.db.GenericDao;
import javax.persistence.Column;
import javax.persistence.Entity;
@ -30,11 +31,9 @@ import javax.persistence.Id;
import javax.persistence.Table;
import javax.persistence.TableGenerator;
import javax.persistence.Transient;
import com.cloud.network.Network.Provider;
import com.cloud.org.Grouping;
import com.cloud.utils.NumbersUtil;
import com.cloud.utils.db.GenericDao;
import java.util.Date;
import java.util.Map;
import java.util.UUID;
@Entity
@Table(name = "data_center")
@ -456,4 +455,9 @@ public class DataCenterVO implements DataCenter {
public void setIp6Dns2(String ip6Dns2) {
this.ip6Dns2 = ip6Dns2;
}
@Override
public PartitionType partitionType() {
return PartitionType.Zone;
}
}

38
engine/schema/src/com/cloud/host/HostVO.java
View File

@ -16,11 +16,12 @@
// under the License.
package com.cloud.host;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import com.cloud.agent.api.VgpuTypesInfo;
import com.cloud.hypervisor.Hypervisor.HypervisorType;
import com.cloud.resource.ResourceState;
import com.cloud.storage.Storage.StoragePoolType;
import com.cloud.utils.NumbersUtil;
import com.cloud.utils.db.GenericDao;
import javax.persistence.Column;
import javax.persistence.DiscriminatorColumn;
@ -37,13 +38,11 @@ import javax.persistence.Table;
import javax.persistence.Temporal;
import javax.persistence.TemporalType;
import javax.persistence.Transient;
import com.cloud.agent.api.VgpuTypesInfo;
import com.cloud.hypervisor.Hypervisor.HypervisorType;
import com.cloud.resource.ResourceState;
import com.cloud.storage.Storage.StoragePoolType;
import com.cloud.utils.NumbersUtil;
import com.cloud.utils.db.GenericDao;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
@Entity
@Table(name = "host")
@ -178,6 +177,11 @@ public class HostVO implements Host {
return clusterId;
}
@Override
public ResourceType resourceType() {
return ResourceType.Host;
}
public void setClusterId(Long clusterId) {
this.clusterId = clusterId;
}
@ -715,6 +719,11 @@ public class HostVO implements Host {
return (getResourceState() == ResourceState.Maintenance || getResourceState() == ResourceState.ErrorInMaintenance || getResourceState() == ResourceState.PrepareForMaintenance);
}
@Override
public boolean isDisabled() {
return (getResourceState() == ResourceState.Disabled);
}
public long getUpdated() {
return updated;
}
@ -732,4 +741,9 @@ public class HostVO implements Host {
public void setUuid(String uuid) {
this.uuid = uuid;
}
@Override
public PartitionType partitionType() {
return PartitionType.Host;
}
}

33
engine/schema/src/com/cloud/vm/VMInstanceVO.java
View File

@ -16,11 +16,14 @@
// under the License.
package com.cloud.vm;
import java.security.NoSuchAlgorithmException;
import java.security.SecureRandom;
import java.util.Date;
import java.util.Map;
import java.util.UUID;
import com.cloud.hypervisor.Hypervisor.HypervisorType;
import com.cloud.utils.db.Encrypt;
import com.cloud.utils.db.GenericDao;
import com.cloud.utils.db.StateMachine;
import com.cloud.utils.fsm.FiniteStateObject;
import com.cloud.vm.VirtualMachine.State;
import org.apache.commons.codec.binary.Base64;
import org.apache.log4j.Logger;
import javax.persistence.Column;
import javax.persistence.DiscriminatorColumn;
@ -36,16 +39,11 @@ import javax.persistence.TableGenerator;
import javax.persistence.Temporal;
import javax.persistence.TemporalType;
import javax.persistence.Transient;
import org.apache.commons.codec.binary.Base64;
import org.apache.log4j.Logger;
import com.cloud.hypervisor.Hypervisor.HypervisorType;
import com.cloud.utils.db.Encrypt;
import com.cloud.utils.db.GenericDao;
import com.cloud.utils.db.StateMachine;
import com.cloud.utils.fsm.FiniteStateObject;
import com.cloud.vm.VirtualMachine.State;
import java.security.NoSuchAlgorithmException;
import java.security.SecureRandom;
import java.util.Date;
import java.util.Map;
import java.util.UUID;
@Entity
@Table(name = "vm_instance")
@ -562,4 +560,9 @@ public class VMInstanceVO implements VirtualMachine, FiniteStateObject<State, Vi
public void setPowerHostId(Long hostId) {
powerHostId = hostId;
}
@Override
public PartitionType partitionType() {
return PartitionType.VM;
}
}

5
engine/schema/src/org/apache/cloudstack/engine/cloud/entity/api/db/VMEntityVO.java
View File

@ -550,4 +550,9 @@ public class VMEntityVO implements VirtualMachine, FiniteStateObject<State, Virt
public void setDisplay(boolean display) {
this.display = display;
}
@Override
public PartitionType partitionType() {
return PartitionType.VM;
}
}

142
engine/schema/src/org/apache/cloudstack/ha/HAConfigVO.java Normal file
View File

@ -0,0 +1,142 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.ha;
import com.cloud.utils.db.StateMachine;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.EnumType;
import javax.persistence.Enumerated;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.Table;
import javax.persistence.Temporal;
import javax.persistence.TemporalType;
import java.util.Date;
@Entity
@Table(name = "ha_config")
public class HAConfigVO implements HAConfig {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
@Column(name = "id")
private Long id;
@Column(name = "resource_id", updatable = false, nullable = false)
private long resourceId;
@Column(name = "resource_type", nullable = false)
@Enumerated(value = EnumType.STRING)
private HAResource.ResourceType resourceType;
@Column(name = "enabled")
private boolean enabled = false;
// There is no setter for status because it has to be set in the dao code
@Enumerated(value = EnumType.STRING)
@StateMachine(state = HAState.class, event = HAConfig.Event.class)
@Column(name = "ha_state", updatable = true, nullable = false, length = 32)
private HAState haState = null;
@Column(name = "provider")
private String haProvider;
// This field should be updated every time the state is updated.
// There's no set method in the vo object because it is done with in the dao code.
@Column(name = "update_count", updatable = true, nullable = false)
private long updateCount;
@Column(name = "update_time", updatable = true)
@Temporal(value = TemporalType.TIMESTAMP)
private Date updateTime;
@Column(name = "mgmt_server_id")
private Long managementServerId;
public HAConfigVO() {
}
@Override
public long getId() {
return id;
}
public long getResourceId() {
return resourceId;
}
public HAResource.ResourceType getResourceType() {
return resourceType;
}
public boolean isEnabled() {
return enabled;
}
public long getUpdateCount() {
return updateCount;
}
public long incrUpdateCount() {
updateCount++;
return updateCount;
}
public Date getUpdateTime() {
return updateTime;
}
public Long getManagementServerId() {
return managementServerId;
}
public HAState getHaState() {
return haState;
}
@Override
public HAState getState() {
return haState;
}
public String getHaProvider() {
return haProvider;
}
public void setHaProvider(String haProvider) {
this.haProvider = haProvider;
}
public void setResourceId(long resourceId) {
this.resourceId = resourceId;
}
public void setResourceType(HAResource.ResourceType resourceType) {
this.resourceType = resourceType;
}
public void setEnabled(boolean enabled) {
this.enabled = enabled;
}
public void setManagementServerId(Long managementServerId) {
this.managementServerId = managementServerId;
}
}

32
engine/schema/src/org/apache/cloudstack/ha/dao/HAConfigDao.java Normal file
View File

@ -0,0 +1,32 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.ha.dao;
import com.cloud.utils.db.GenericDao;
import com.cloud.utils.fsm.StateDao;
import org.apache.cloudstack.ha.HAConfig;
import org.apache.cloudstack.ha.HAConfigVO;
import org.apache.cloudstack.ha.HAResource;
import java.util.List;
public interface HAConfigDao extends GenericDao<HAConfigVO, Long>, StateDao<HAConfig.HAState, HAConfig.Event, HAConfig> {
HAConfig findHAResource(long resourceId, HAResource.ResourceType resourceType);
List<HAConfig> listHAResource(final Long resourceId, final HAResource.ResourceType resourceType);
void expireServerOwnership(long serverId);
}

149
engine/schema/src/org/apache/cloudstack/ha/dao/HAConfigDaoImpl.java Normal file
View File

@ -0,0 +1,149 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.ha.dao;
import com.cloud.utils.DateUtil;
import com.cloud.utils.db.Attribute;
import com.cloud.utils.db.DB;
import com.cloud.utils.db.GenericDaoBase;
import com.cloud.utils.db.SearchBuilder;
import com.cloud.utils.db.SearchCriteria;
import com.cloud.utils.db.Transaction;
import com.cloud.utils.db.TransactionCallbackNoReturn;
import com.cloud.utils.db.TransactionLegacy;
import com.cloud.utils.db.TransactionStatus;
import com.cloud.utils.db.UpdateBuilder;
import org.apache.cloudstack.ha.HAConfig;
import org.apache.cloudstack.ha.HAConfigVO;
import org.apache.cloudstack.ha.HAResource;
import org.apache.log4j.Logger;
import org.springframework.stereotype.Component;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
@DB
@Component
public class HAConfigDaoImpl extends GenericDaoBase<HAConfigVO, Long> implements HAConfigDao {
private static final Logger LOG = Logger.getLogger(HAConfigDaoImpl.class);
private static final String EXPIRE_OWNERSHIP = "UPDATE ha_config set mgmt_server_id=NULL where mgmt_server_id=?";
private SearchBuilder<HAConfigVO> ResourceSearch;
private SearchBuilder<HAConfigVO> StateUpdateSearch;
private Attribute HAStateAttr;
private Attribute MsIdAttr;
private Attribute UpdateTimeAttr;
public HAConfigDaoImpl() {
super();
ResourceSearch = createSearchBuilder();
ResourceSearch.and("resourceId", ResourceSearch.entity().getResourceId(), SearchCriteria.Op.EQ);
ResourceSearch.and("resourceType", ResourceSearch.entity().getResourceType(), SearchCriteria.Op.EQ);
ResourceSearch.done();
StateUpdateSearch = createSearchBuilder();
StateUpdateSearch.and("id", StateUpdateSearch.entity().getId(), SearchCriteria.Op.EQ);
StateUpdateSearch.and("haState", StateUpdateSearch.entity().getHaState(), SearchCriteria.Op.EQ);
StateUpdateSearch.and("update", StateUpdateSearch.entity().getUpdateCount(), SearchCriteria.Op.EQ);
StateUpdateSearch.done();
HAStateAttr = _allAttributes.get("haState");
MsIdAttr = _allAttributes.get("managementServerId");
UpdateTimeAttr = _allAttributes.get("updateTime");
assert (HAStateAttr != null && MsIdAttr != null && UpdateTimeAttr != null) : "Couldn't find one of these attributes";
}
@Override
public boolean updateState(HAConfig.HAState currentState, HAConfig.Event event, HAConfig.HAState nextState, HAConfig vo, Object data) {
HAConfigVO haConfig = (HAConfigVO) vo;
if (haConfig == null) {
if (LOG.isTraceEnabled()) {
LOG.trace("Invalid ha config view object provided");
}
return false;
}
Long newManagementServerId = event.getServerId();
if (currentState == nextState && (haConfig.getManagementServerId() != null && haConfig.getManagementServerId().equals(newManagementServerId))) {
return false;
}
if (event == HAConfig.Event.Disabled) {
newManagementServerId = null;
}
SearchCriteria<HAConfigVO> sc = StateUpdateSearch.create();
sc.setParameters("id", haConfig.getId());
sc.setParameters("haState", currentState);
sc.setParameters("update", haConfig.getUpdateCount());
haConfig.incrUpdateCount();
UpdateBuilder ub = getUpdateBuilder(haConfig);
ub.set(haConfig, HAStateAttr, nextState);
ub.set(haConfig, UpdateTimeAttr, DateUtil.currentGMTTime());
ub.set(haConfig, MsIdAttr, newManagementServerId);
int result = update(ub, sc, null);
if (LOG.isTraceEnabled() && result <= 0) {
LOG.trace(String.format("Failed to update HA state from:%s to:%s due to event:%s for the ha_config id:%d", currentState, nextState, event, haConfig.getId()));
}
return result > 0;
}
@Override
public HAConfig findHAResource(final long resourceId, final HAResource.ResourceType resourceType) {
final SearchCriteria<HAConfigVO> sc = ResourceSearch.create();
sc.setParameters("resourceId", resourceId);
sc.setParameters("resourceType", resourceType);
return findOneBy(sc);
}
@Override
public List<HAConfig> listHAResource(final Long resourceId, final HAResource.ResourceType resourceType) {
final SearchCriteria<HAConfigVO> sc = ResourceSearch.create();
if (resourceId != null && resourceId > 0L) {
sc.setParameters("resourceId", resourceId);
}
if (resourceType != null) {
sc.setParameters("resourceType", resourceType);
}
return new ArrayList<HAConfig>(listBy(sc));
}
@Override
public void expireServerOwnership(final long serverId) {
Transaction.execute(new TransactionCallbackNoReturn() {
@Override
public void doInTransactionWithoutResult(TransactionStatus status) {
TransactionLegacy txn = TransactionLegacy.currentTxn();
try (final PreparedStatement pstmt = txn.prepareAutoCloseStatement(EXPIRE_OWNERSHIP);) {
pstmt.setLong(1, serverId);
pstmt.executeUpdate();
} catch (SQLException e) {
txn.rollback();
LOG.warn("Failed to expire HA ownership of management server id: " + serverId);
}
}
});
}
}

2
engine/schema/src/org/apache/cloudstack/outofbandmanagement/dao/OutOfBandManagementDao.java
View File

@ -27,5 +27,5 @@ import java.util.List;
public interface OutOfBandManagementDao extends GenericDao<OutOfBandManagementVO, Long>, StateDao<OutOfBandManagement.PowerState, OutOfBandManagement.PowerState.Event, OutOfBandManagement> {
OutOfBandManagement findByHost(long hostId);
List<OutOfBandManagementVO> findAllByManagementServer(long serverId);
void expireOutOfBandManagementOwnershipByServer(long serverId);
void expireServerOwnership(long serverId);
}

2
engine/schema/src/org/apache/cloudstack/outofbandmanagement/dao/OutOfBandManagementDaoImpl.java
View File

@ -110,7 +110,7 @@ public class OutOfBandManagementDaoImpl extends GenericDaoBase<OutOfBandManageme
}
@Override
public void expireOutOfBandManagementOwnershipByServer(long serverId) {
public void expireServerOwnership(long serverId) {
final String resetOwnerSql = "UPDATE oobm set mgmt_server_id=NULL, power_state=NULL where mgmt_server_id=?";
executeExpireOwnershipSql(resetOwnerSql, serverId);
if (LOG.isDebugEnabled()) {

7
plugins/hypervisors/kvm/resources/META-INF/cloudstack/kvm-compute/spring-kvm-compute-context.xml
View File

@ -30,5 +30,12 @@
<bean id="KVMInvestigator" class="com.cloud.ha.KVMInvestigator">
<property name="name" value="KVMInvestigator" />
</bean>
<bean id="KVMHostActivityChecker" class="org.apache.cloudstack.kvm.ha.KVMHostActivityChecker">
</bean>
<bean id="KVMHAProvider" class="org.apache.cloudstack.kvm.ha.KVMHAProvider">
<property name="name" value="KVMHAProvider" />
</bean>
</beans>

18
plugins/hypervisors/kvm/src/com/cloud/ha/KVMInvestigator.java
View File

@ -1,4 +1,5 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@ -29,7 +30,8 @@ import com.cloud.hypervisor.Hypervisor;
import com.cloud.resource.ResourceManager;
import com.cloud.storage.Storage.StoragePoolType;
import com.cloud.utils.component.AdapterBase;
import com.cloud.vm.VirtualMachine;
import org.apache.cloudstack.ha.HAManager;
import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
import org.apache.log4j.Logger;
@ -49,10 +51,17 @@ public class KVMInvestigator extends AdapterBase implements Investigator {
ResourceManager _resourceMgr;
@Inject
PrimaryDataStoreDao _storagePoolDao;
@Inject
private HAManager haManager;
@Override
public Boolean isVmAlive(com.cloud.vm.VirtualMachine vm, Host host) {
public Boolean isVmAlive(VirtualMachine vm, Host host) {
if (haManager.isHAEligible(host)) {
return haManager.isVMAliveOnHost(host);
}
Status status = isAgentAlive(host);
s_logger.debug("HA: HOST is ineligible legacy state " + status + " for host " + host.getId());
if (status == null) {
return null;
}
@ -65,6 +74,10 @@ public class KVMInvestigator extends AdapterBase implements Investigator {
return null;
}
if (haManager.isHAEligible(agent)) {
return haManager.getHostStatus(agent);
}
List<StoragePoolVO> clusterPools = _storagePoolDao.listPoolsByCluster(agent.getClusterId());
boolean hasNfs = false;
for (StoragePoolVO pool : clusterPools) {
@ -119,6 +132,7 @@ public class KVMInvestigator extends AdapterBase implements Investigator {
if (neighbourStatus == Status.Down && (hostStatus == Status.Disconnected || hostStatus == Status.Down)) {
hostStatus = Status.Down;
}
s_logger.debug("HA: HOST is ineligible legacy state " + hostStatus + " for host " + agent.getId());
return hostStatus;
}
}

5
plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/KVMHABase.java
View File

@ -179,4 +179,9 @@ public class KVMHABase {
return result;
}
public Boolean checkingHB() {
// TODO Auto-generated method stub
return null;
}
}

14
plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/KVMHAChecker.java
View File

@ -17,14 +17,18 @@
package com.cloud.hypervisor.kvm.resource;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Callable;
import javax.ejb.Local;
import org.apache.log4j.Logger;
import com.cloud.utils.script.OutputInterpreter;
import com.cloud.utils.script.Script;
@Local(value = {KVMHAChecker.class})
public class KVMHAChecker extends KVMHABase implements Callable<Boolean> {
private static final Logger s_logger = Logger.getLogger(KVMHAChecker.class);
private List<NfsStoragePool> _pools;
@ -40,10 +44,10 @@ public class KVMHAChecker extends KVMHABase implements Callable<Boolean> {
* True means heartbeaing is on going, or we can't get it's status. False
* means heartbeating is stopped definitely
*/
private Boolean checkingHB() {
@Override
public Boolean checkingHB() {
List<Boolean> results = new ArrayList<Boolean>();
for (NfsStoragePool pool : _pools) {
Script cmd = new Script(s_heartBeatPath, _heartBeatCheckerTimeout, s_logger);
cmd.add("-i", pool._poolIp);
cmd.add("-p", pool._poolMountSourcePath);
@ -53,9 +57,9 @@ public class KVMHAChecker extends KVMHABase implements Callable<Boolean> {
cmd.add("-t", String.valueOf(_heartBeatUpdateFreq / 1000));
OutputInterpreter.OneLineParser parser = new OutputInterpreter.OneLineParser();
String result = cmd.execute(parser);
s_logger.debug("pool: " + pool._poolIp);
s_logger.debug("reture: " + result);
s_logger.debug("parser: " + parser.getLine());
s_logger.debug("KVMHAChecker pool: " + pool._poolIp);
s_logger.debug("KVMHAChecker result: " + result);
s_logger.debug("KVMHAChecker parser: " + parser.getLine());
if (result == null && parser.getLine().contains("> DEAD <")) {
s_logger.debug("read heartbeat failed: " + result);
results.add(false);

6
plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java
View File

@ -69,6 +69,12 @@ public class KVMHAMonitor extends KVMHABase implements Runnable {
}
}
public NfsStoragePool getStoragePool(String uuid) {
synchronized (_storagePool) {
return _storagePool.get(uuid);
}
}
private class Monitor extends ManagedContextRunnable {
@Override

70
plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/KVMHAVMActivityChecker.java Normal file
View File

@ -0,0 +1,70 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package com.cloud.hypervisor.kvm.resource;
import com.cloud.utils.script.OutputInterpreter;
import com.cloud.utils.script.Script;
import org.apache.log4j.Logger;
import org.joda.time.Duration;
import java.util.concurrent.Callable;
public class KVMHAVMActivityChecker extends KVMHABase implements Callable<Boolean> {
private static final Logger LOG = Logger.getLogger(KVMHAVMActivityChecker.class);
final private NfsStoragePool nfsStoragePool;
final private String hostIP;
final private String volumeUuidList;
final private String vmActivityCheckPath;
final private Duration activityScriptTimeout = Duration.standardSeconds(3600L);
final private long suspectTimeInSeconds;
public KVMHAVMActivityChecker(final NfsStoragePool pool, final String host, final String volumeUUIDListString, String vmActivityCheckPath, final long suspectTime) {
this.nfsStoragePool = pool;
this.hostIP = host;
this.volumeUuidList = volumeUUIDListString;
this.vmActivityCheckPath = vmActivityCheckPath;
this.suspectTimeInSeconds = suspectTime;
}
@Override
public Boolean checkingHB() {
Script cmd = new Script(vmActivityCheckPath, activityScriptTimeout.getStandardSeconds(), LOG);
cmd.add("-i", nfsStoragePool._poolIp);
cmd.add("-p", nfsStoragePool._poolMountSourcePath);
cmd.add("-m", nfsStoragePool._mountDestPath);
cmd.add("-h", hostIP);
cmd.add("-u", volumeUuidList);
cmd.add("-t", String.valueOf(String.valueOf(System.currentTimeMillis() / 1000)));
cmd.add("-d", String.valueOf(suspectTimeInSeconds));
OutputInterpreter.OneLineParser parser = new OutputInterpreter.OneLineParser();
String result = cmd.execute(parser);
LOG.debug("KVMHAVMActivityChecker pool: " + nfsStoragePool._poolIp);
LOG.debug("KVMHAVMActivityChecker result: " + result);
LOG.debug("KVMHAVMActivityChecker parser: " + parser.getLine());
if (result == null && parser.getLine().contains("DEAD")) {
return false;
} else {
return true;
}
}
@Override
public Boolean call() throws Exception {
return checkingHB();
}
}

39
plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java
View File

@ -105,6 +105,7 @@ import com.cloud.agent.api.CheckNetworkAnswer;
import com.cloud.agent.api.CheckNetworkCommand;
import com.cloud.agent.api.CheckOnHostCommand;
import com.cloud.agent.api.CheckStateCommand;
import com.cloud.agent.api.CheckVMActivityOnStoragePoolCommand;
import com.cloud.agent.api.CheckVirtualMachineAnswer;
import com.cloud.agent.api.CheckVirtualMachineCommand;
import com.cloud.agent.api.CleanupNetworkRulesCmd;
@ -307,6 +308,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
private String _resizeVolumePath;
private String _createTmplPath;
private String _heartBeatPath;
private String _vmActivityCheckPath;
private String _securityGroupPath;
private String _ovsPvlanDhcpHostPath;
private String _ovsPvlanVmPath;
@ -668,6 +670,11 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
throw new ConfigurationException("Unable to find the resizevolume.sh");
}
_vmActivityCheckPath = Script.findScript(kvmScriptsDir, "kvmvmactivity.sh");
if (_vmActivityCheckPath == null) {
throw new ConfigurationException("Unable to find kvmvmactivity.sh");
}
_createTmplPath = Script.findScript(storageScriptsDir, "createtmplt.sh");
if (_createTmplPath == null) {
throw new ConfigurationException("Unable to find the createtmplt.sh");
@ -1396,6 +1403,8 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
return execute((PvlanSetupCommand)cmd);
} else if (cmd instanceof CheckOnHostCommand) {
return execute((CheckOnHostCommand)cmd);
} else if (cmd instanceof CheckVMActivityOnStoragePoolCommand) {
return execute((CheckVMActivityOnStoragePoolCommand)cmd);
} else if (cmd instanceof OvsFetchInterfaceCommand) {
return execute((OvsFetchInterfaceCommand)cmd);
} else if (cmd instanceof OvsSetupBridgeCommand) {
@ -1411,7 +1420,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
} else if (cmd instanceof OvsVpcRoutingPolicyConfigCommand) {
return execute((OvsVpcRoutingPolicyConfigCommand) cmd);
} else {
s_logger.warn("Unsupported command ");
s_logger.warn("Unsupported command " + cmd.getClass());
return Answer.createUnsupportedCommandAnswer(cmd);
}
} catch (final IllegalArgumentException e) {
@ -1742,17 +1751,39 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
try {
Boolean result = future.get();
if (result) {
return new Answer(cmd, false, "Heart is still beating...");
return new Answer(cmd, false, "Heart is beating...");
} else {
s_logger.warn("Heartbeat failed for : " + cmd.getHost().getPrivateNetwork().getIp().toString());
return new Answer(cmd);
}
} catch (InterruptedException e) {
return new Answer(cmd, false, "can't get status of host:");
return new Answer(cmd, false, "CheckOnHostCommand: can't get status of host: InterruptedException");
} catch (ExecutionException e) {
return new Answer(cmd, false, "can't get status of host:");
return new Answer(cmd, false, "CheckOnHostCommand: can't get status of host: ExecutionException");
}
}
protected Answer execute(CheckVMActivityOnStoragePoolCommand cmd) {
ExecutorService executors = Executors.newSingleThreadExecutor();
StorageFilerTO pool = cmd.getPool();
if (StoragePoolType.NetworkFilesystem == pool.getType()){
NfsStoragePool nfspool = _monitor.getStoragePool(pool.getUuid());
KVMHAVMActivityChecker ha = new KVMHAVMActivityChecker(nfspool, cmd.getHost().getPrivateNetwork().getIp(), cmd.getVolumeList(), _vmActivityCheckPath, cmd.getSuspectTimeInSeconds());
Future<Boolean> future = executors.submit(ha);
try {
Boolean result = future.get();
if (result) {
return new Answer(cmd, false, "VMHA disk activity detected ...");
} else {
return new Answer(cmd);
}
} catch (InterruptedException e) {
return new Answer(cmd, false, "CheckVMActivityOnStoragePoolCommand: can't get status of host: InterruptedException");
} catch (ExecutionException e) {
return new Answer(cmd, false, "CheckVMActivityOnStoragePoolCommand: can't get status of host: ExecutionException");
}
}
return new Answer(cmd, false, "Unsupported Storage");
}
protected Storage.StorageResourceType getStorageResourceType() {

56
plugins/hypervisors/kvm/src/org/apache/cloudstack/kvm/ha/KVMHAConfig.java Normal file
View File

@ -0,0 +1,56 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.kvm.ha;
import org.apache.cloudstack.framework.config.ConfigKey;
public class KVMHAConfig {
public static final ConfigKey<Long> KvmHAHealthCheckTimeout = new ConfigKey<>("Advanced", Long.class, "kvm.ha.health.check.timeout", "10",
"The maximum length of time, in seconds, expected for an health check to complete.", true, ConfigKey.Scope.Cluster);
public static final ConfigKey<Long> KvmHAActivityCheckTimeout = new ConfigKey<>("Advanced", Long.class, "kvm.ha.activity.check.timeout", "60",
"The maximum length of time, in seconds, expected for an activity check to complete.", true, ConfigKey.Scope.Cluster);
public static final ConfigKey<Long> KvmHAActivityCheckInterval = new ConfigKey<>("Advanced", Long.class, "kvm.ha.activity.check.interval", "60",
"The interval, in seconds, between activity checks.", true, ConfigKey.Scope.Cluster);
public static final ConfigKey<Long> KvmHAActivityCheckMaxAttempts = new ConfigKey<>("Advanced", Long.class, "kvm.ha.activity.check.max.attempts", "10",
"The maximum number of activity check attempts to perform before deciding to recover or degrade a resource.", true, ConfigKey.Scope.Cluster);
public static final ConfigKey<Double> KvmHAActivityCheckFailureThreshold = new ConfigKey<>("Advanced", Double.class, "kvm.ha.activity.check.failure.ratio", "0.7",
"The activity check failure threshold ratio. This is used with the activity check maximum attempts for deciding to recover or degrade a resource. For most environments, please keep this value above 0.5.",
true, ConfigKey.Scope.Cluster);
public static final ConfigKey<Long> KvmHADegradedMaxPeriod = new ConfigKey<>("Advanced", Long.class, "kvm.ha.degraded.max.period", "300",
"The maximum length of time, in seconds, a resource can be in degraded state where only health checks are performed.", true, ConfigKey.Scope.Cluster);
public static final ConfigKey<Long> KvmHARecoverTimeout = new ConfigKey<>("Advanced", Long.class, "kvm.ha.recover.timeout", "60",
"The maximum length of time, in seconds, expected for a recovery operation to complete.", true, ConfigKey.Scope.Cluster);
public static final ConfigKey<Long> KvmHARecoverWaitPeriod = new ConfigKey<>("Advanced", Long.class, "kvm.ha.recover.wait.period", "600",
"The maximum length of time, in seconds, to wait for a resource to recover.", true, ConfigKey.Scope.Cluster);
public static final ConfigKey<Long> KvmHARecoverAttemptThreshold = new ConfigKey<>("Advanced", Long.class, "kvm.ha.recover.failure.threshold", "1",
"The maximum recovery attempts to be made for a resource, after which the resource is fenced. The recovery counter resets when a health check passes for a resource.",
true, ConfigKey.Scope.Cluster);
public static final ConfigKey<Long> KvmHAFenceTimeout = new ConfigKey<>("Advanced", Long.class, "kvm.ha.fence.timeout", "60",
"The maximum length of time, in seconds, expected for a fence operation to complete.", true, ConfigKey.Scope.Cluster);
}

157
plugins/hypervisors/kvm/src/org/apache/cloudstack/kvm/ha/KVMHAProvider.java Normal file
View File

@ -0,0 +1,157 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.cloudstack.kvm.ha;
import com.cloud.host.Host;
import com.cloud.hypervisor.Hypervisor;
import org.apache.cloudstack.api.response.OutOfBandManagementResponse;
import org.apache.cloudstack.framework.config.ConfigKey;
import org.apache.cloudstack.framework.config.Configurable;
import org.apache.cloudstack.ha.HAResource;
import org.apache.cloudstack.ha.provider.HACheckerException;
import org.apache.cloudstack.ha.provider.HAFenceException;
import org.apache.cloudstack.ha.provider.HAProvider;
import org.apache.cloudstack.ha.provider.HARecoveryException;
import org.apache.cloudstack.ha.provider.host.HAAbstractHostProvider;
import org.apache.cloudstack.outofbandmanagement.OutOfBandManagement.PowerOperation;
import org.apache.cloudstack.outofbandmanagement.OutOfBandManagementService;
import org.apache.log4j.Logger;
import org.joda.time.DateTime;
import javax.inject.Inject;
import java.security.InvalidParameterException;
public final class KVMHAProvider extends HAAbstractHostProvider implements HAProvider<Host>, Configurable {
private final static Logger LOG = Logger.getLogger(KVMHAProvider.class);
@Inject
protected KVMHostActivityChecker hostActivityChecker;
@Inject
protected OutOfBandManagementService outOfBandManagementService;
@Override
public boolean isEligible(final Host host) {
if (outOfBandManagementService.isOutOfBandManagementEnabled(host)){
return !isInMaintenanceMode(host) && !isDisabled(host) &&
hostActivityChecker.getNeighbors(host).length > 0 &&
(Hypervisor.HypervisorType.KVM.equals(host.getHypervisorType()) ||
Hypervisor.HypervisorType.LXC.equals(host.getHypervisorType()));
}
return false;
}
@Override
public boolean isHealthy(final Host r) throws HACheckerException {
return hostActivityChecker.isHealthy(r);
}
@Override
public boolean hasActivity(final Host r, final DateTime suspectTime) throws HACheckerException {
return hostActivityChecker.isActive(r, suspectTime);
}
@Override
public boolean recover(Host r) throws HARecoveryException {
try {
if (outOfBandManagementService.isOutOfBandManagementEnabled(r)){
final OutOfBandManagementResponse resp = outOfBandManagementService.executePowerOperation(r, PowerOperation.RESET, null);
return resp.getSuccess();
} else {
LOG.warn("OOBM recover operation failed for the host " + r.getName());
return false;
}
} catch (Exception e){
LOG.warn("OOBM service is not configured or enabled for this host " + r.getName() + " error is " + e.getMessage());
throw new HARecoveryException(" OOBM service is not configured or enabled for this host " + r.getName(), e);
}
}
@Override
public boolean fence(Host r) throws HAFenceException {
try {
if (outOfBandManagementService.isOutOfBandManagementEnabled(r)){
final OutOfBandManagementResponse resp = outOfBandManagementService.executePowerOperation(r, PowerOperation.OFF, null);
return resp.getSuccess();
} else {
LOG.warn("OOBM fence operation failed for this host " + r.getName());
return false;
}
} catch (Exception e){
LOG.warn("OOBM service is not configured or enabled for this host " + r.getName() + " error is " + e.getMessage());
throw new HAFenceException("OOBM service is not configured or enabled for this host " + r.getName() , e);
}
}
@Override
public HAResource.ResourceSubType resourceSubType() {
return HAResource.ResourceSubType.KVM;
}
@Override
public Object getConfigValue(final HAProviderConfig name, final Host host) {
final Long clusterId = host.getClusterId();
switch (name) {
case HealthCheckTimeout:
return KVMHAConfig.KvmHAHealthCheckTimeout.valueIn(clusterId);
case ActivityCheckTimeout:
return KVMHAConfig.KvmHAActivityCheckTimeout.valueIn(clusterId);
case MaxActivityCheckInterval:
return KVMHAConfig.KvmHAActivityCheckInterval.valueIn(clusterId);
case MaxActivityChecks:
return KVMHAConfig.KvmHAActivityCheckMaxAttempts.valueIn(clusterId);
case ActivityCheckFailureRatio:
return KVMHAConfig.KvmHAActivityCheckFailureThreshold.valueIn(clusterId);
case RecoveryWaitTimeout:
return KVMHAConfig.KvmHARecoverWaitPeriod.valueIn(clusterId);
case RecoveryTimeout:
return KVMHAConfig.KvmHARecoverTimeout.valueIn(clusterId);
case FenceTimeout:
return KVMHAConfig.KvmHAFenceTimeout.valueIn(clusterId);
case MaxRecoveryAttempts:
return KVMHAConfig.KvmHARecoverAttemptThreshold.valueIn(clusterId);
case MaxDegradedWaitTimeout:
return KVMHAConfig.KvmHADegradedMaxPeriod.valueIn(clusterId);
default:
throw new InvalidParameterException("Unknown HAProviderConfig " + name.toString());
}
}
@Override
public String getConfigComponentName() {
return KVMHAConfig.class.getSimpleName();
}
@Override
public ConfigKey<?>[] getConfigKeys() {
return new ConfigKey<?>[] {
KVMHAConfig.KvmHAHealthCheckTimeout,
KVMHAConfig.KvmHAActivityCheckTimeout,
KVMHAConfig.KvmHARecoverTimeout,
KVMHAConfig.KvmHAFenceTimeout,
KVMHAConfig.KvmHAActivityCheckInterval,
KVMHAConfig.KvmHAActivityCheckMaxAttempts,
KVMHAConfig.KvmHAActivityCheckFailureThreshold,
KVMHAConfig.KvmHADegradedMaxPeriod,
KVMHAConfig.KvmHARecoverWaitPeriod,
KVMHAConfig.KvmHARecoverAttemptThreshold
};
}
}

205
plugins/hypervisors/kvm/src/org/apache/cloudstack/kvm/ha/KVMHostActivityChecker.java Normal file
View File

@ -0,0 +1,205 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.kvm.ha;
import com.cloud.agent.AgentManager;
import com.cloud.agent.api.Answer;
import com.cloud.agent.api.CheckOnHostCommand;
import com.cloud.agent.api.CheckVMActivityOnStoragePoolCommand;
import com.cloud.exception.StorageUnavailableException;
import com.cloud.host.Host;
import com.cloud.host.HostVO;
import com.cloud.host.Status;
import com.cloud.hypervisor.Hypervisor;
import com.cloud.resource.ResourceManager;
import com.cloud.storage.StorageManager;
import com.cloud.storage.StoragePool;
import com.cloud.storage.Volume;
import com.cloud.storage.VolumeVO;
import com.cloud.storage.dao.VolumeDao;
import com.cloud.utils.component.AdapterBase;
import com.cloud.vm.VMInstanceVO;
import com.cloud.vm.VirtualMachine;
import com.cloud.vm.dao.VMInstanceDao;
import org.apache.cloudstack.ha.provider.ActivityCheckerInterface;
import org.apache.cloudstack.ha.provider.HACheckerException;
import org.apache.cloudstack.ha.provider.HealthCheckerInterface;
import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
import org.apache.commons.lang.ArrayUtils;
import org.apache.log4j.Logger;
import javax.inject.Inject;
import java.util.ArrayList;
import org.joda.time.DateTime;
import java.util.HashMap;
import java.util.List;
public class KVMHostActivityChecker extends AdapterBase implements ActivityCheckerInterface<Host>, HealthCheckerInterface<Host> {
private final static Logger LOG = Logger.getLogger(KVMHostActivityChecker.class);
@Inject
private VolumeDao volumeDao;
@Inject
private VMInstanceDao vmInstanceDao;
@Inject
private AgentManager agentMgr;
@Inject
private PrimaryDataStoreDao storagePool;
@Inject
private StorageManager storageManager;
@Inject
private ResourceManager resourceManager;
@Override
public boolean isActive(Host r, DateTime suspectTime) throws HACheckerException {
try {
return isVMActivtyOnHost(r, suspectTime);
}
catch (StorageUnavailableException e){
throw new HACheckerException("Storage is unavailable to do the check, mostly host is not reachable ", e);
}
catch (Exception e){
throw new HACheckerException("Operation timed out, mostly host is not reachable ", e);
}
}
@Override
public boolean isHealthy(Host r) {
return isAgentActive(r);
}
private boolean isAgentActive(Host agent) {
if (agent.getHypervisorType() != Hypervisor.HypervisorType.KVM && agent.getHypervisorType() != Hypervisor.HypervisorType.LXC) {
throw new IllegalStateException("Calling KVM investigator for non KVM Host of type " + agent.getHypervisorType());
}
Status hostStatus = Status.Unknown;
Status neighbourStatus = Status.Unknown;
final CheckOnHostCommand cmd = new CheckOnHostCommand(agent);
try {
Answer answer = agentMgr.easySend(agent.getId(), cmd);
if (answer != null) {
hostStatus = answer.getResult() ? Status.Down : Status.Up;
if ( hostStatus == Status.Up ){
return true;
}
}
else {
hostStatus = Status.Disconnected;
}
} catch (Exception e) {
LOG.warn("Failed to send command to host: " + agent.getId());
}
List<HostVO> neighbors = resourceManager.listHostsInClusterByStatus(agent.getClusterId(), Status.Up);
for (HostVO neighbor : neighbors) {
if (neighbor.getId() == agent.getId() || (neighbor.getHypervisorType() != Hypervisor.HypervisorType.KVM && neighbor.getHypervisorType() != Hypervisor.HypervisorType.LXC)) {
continue;
}
if (LOG.isTraceEnabled()){
LOG.trace("Investigating host:" + agent.getId() + " via neighbouring host:" + neighbor.getId());
}
try {
Answer answer = agentMgr.easySend(neighbor.getId(), cmd);
if (answer != null) {
neighbourStatus = answer.getResult() ? Status.Down : Status.Up;
if (LOG.isTraceEnabled()){
LOG.trace("Neighbouring host:" + neighbor.getId() + " returned status:" + neighbourStatus + " for the investigated host:" + agent.getId());
}
if (neighbourStatus == Status.Up) {
break;
}
}
} catch (Exception e) {
if (LOG.isTraceEnabled()) {
LOG.trace("Failed to send command to host: " + neighbor.getId());
}
}
}
if (neighbourStatus == Status.Up && (hostStatus == Status.Disconnected || hostStatus == Status.Down)) {
hostStatus = Status.Disconnected;
}
if (neighbourStatus == Status.Down && (hostStatus == Status.Disconnected || hostStatus == Status.Down)) {
hostStatus = Status.Down;
}
if (LOG.isTraceEnabled()){
LOG.trace("Resource state = " + hostStatus.name());
}
return hostStatus == Status.Up;
}
private boolean isVMActivtyOnHost(Host agent, DateTime suspectTime) throws StorageUnavailableException {
if (agent.getHypervisorType() != Hypervisor.HypervisorType.KVM && agent.getHypervisorType() != Hypervisor.HypervisorType.LXC) {
throw new IllegalStateException("Calling KVM investigator for non KVM Host of type " + agent.getHypervisorType());
}
boolean activityStatus = true;
HashMap<StoragePool, List<Volume>> poolVolMap = getVolumeUuidOnHost(agent);
for (StoragePool pool : poolVolMap.keySet()) {
//for each storage pool find activity
List<Volume> volume_list = poolVolMap.get(pool);
final CheckVMActivityOnStoragePoolCommand cmd = new CheckVMActivityOnStoragePoolCommand(agent, pool, volume_list, suspectTime);
//send the command to appropriate storage pool
Answer answer = storageManager.sendToPool(pool, getNeighbors(agent), cmd);
if (answer != null) {
activityStatus = ! answer.getResult();
} else {
throw new IllegalStateException("Did not get a valid response for VM activity check for host " + agent.getId());
}
}
if (LOG.isDebugEnabled()){
LOG.debug("Resource active = " + activityStatus);
}
return activityStatus;
}
private HashMap<StoragePool, List<Volume>> getVolumeUuidOnHost(Host agent) {
List<VMInstanceVO> vm_list = vmInstanceDao.listByHostId(agent.getId());
List<VolumeVO> volume_list = new ArrayList<VolumeVO>();
for (VirtualMachine vm : vm_list) {
List<VolumeVO> vm_volume_list = volumeDao.findByInstance(vm.getId());
volume_list.addAll(vm_volume_list);
}
HashMap<StoragePool, List<Volume>> poolVolMap = new HashMap<StoragePool, List<Volume>>();
for (Volume vol : volume_list) {
StoragePool sp = storagePool.findById(vol.getPoolId());
if (!poolVolMap.containsKey(sp)) {
List<Volume> list = new ArrayList<Volume>();
list.add(vol);
poolVolMap.put(sp, list);
} else {
poolVolMap.get(sp).add(vol);
}
}
return poolVolMap;
}
public long[] getNeighbors(Host agent) {
List<Long> neighbors = new ArrayList<Long>();
List<HostVO> cluster_hosts = resourceManager.listHostsInClusterByStatus(agent.getClusterId(), Status.Up);
for (HostVO host : cluster_hosts) {
if (host.getId() == agent.getId() || (host.getHypervisorType() != Hypervisor.HypervisorType.KVM && host.getHypervisorType() != Hypervisor.HypervisorType.LXC)) {
continue;
}
neighbors.add(host.getId());
}
return ArrayUtils.toPrimitive(neighbors.toArray(new Long[neighbors.size()]));
}
}

84
plugins/hypervisors/kvm/test/org/apache/cloudstack/kvm/ha/KVMHostHATest.java Normal file
View File

@ -0,0 +1,84 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.cloudstack.kvm.ha;
import com.cloud.exception.StorageUnavailableException;
import com.cloud.host.Host;
import com.cloud.hypervisor.Hypervisor.HypervisorType;
import org.apache.cloudstack.ha.provider.HACheckerException;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.MockitoAnnotations;
import org.mockito.runners.MockitoJUnitRunner;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.when;
import org.joda.time.DateTime;
@RunWith(MockitoJUnitRunner.class)
public class KVMHostHATest {
@Mock
private Host host;
@Mock
private KVMHostActivityChecker kvmHostActivityChecker;
private KVMHAProvider kvmHAProvider;
@Before
public void setup() {
MockitoAnnotations.initMocks(this);
kvmHAProvider = new KVMHAProvider();
kvmHAProvider.hostActivityChecker = kvmHostActivityChecker;
}
@Test
public void testHostActivityForHealthyHost() throws HACheckerException, StorageUnavailableException {
when(host.getHypervisorType()).thenReturn(HypervisorType.KVM);
when(kvmHostActivityChecker.isHealthy(host)).thenReturn(true);
assertTrue(kvmHAProvider.isHealthy(host));
}
@Test
public void testHostActivityForUnHealthyHost() throws HACheckerException, StorageUnavailableException {
when(host.getHypervisorType()).thenReturn(HypervisorType.KVM);
when(kvmHostActivityChecker.isHealthy(host)).thenReturn(false);
assertFalse(kvmHAProvider.isHealthy(host));
}
@Test
public void testHostActivityForActiveHost() throws HACheckerException, StorageUnavailableException {
when(host.getHypervisorType()).thenReturn(HypervisorType.KVM);
DateTime dt = new DateTime();
when(kvmHostActivityChecker.isActive(host, dt)).thenReturn(true);
assertTrue(kvmHAProvider.hasActivity(host, dt));
}
@Test
public void testHostActivityForDownHost() throws HACheckerException, StorageUnavailableException {
when(host.getHypervisorType()).thenReturn(HypervisorType.KVM);
DateTime dt = new DateTime();
when(kvmHostActivityChecker.isActive(host, dt)).thenReturn(false);
assertFalse(kvmHAProvider.hasActivity(host, dt));
}
}

5
plugins/hypervisors/simulator/pom.xml
View File

@ -63,5 +63,10 @@
<artifactId>cloud-engine-storage-snapshot</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>${cs.guava.version}</version>
</dependency>
</dependencies>
</project>

4
plugins/hypervisors/simulator/resources/META-INF/cloudstack/simulator-compute/spring-simulator-compute-context.xml
View File

@ -36,4 +36,8 @@
<bean id="SimulatorFencer" class="com.cloud.ha.SimulatorFencer">
<property name="name" value="Simulator Fencer"/>
</bean>
<bean id="SimulatorHAProvider" class="org.apache.cloudstack.ha.SimulatorHAProvider">
<property name="name" value="SimulatorHAProvider" />
</bean>
</beans>

4
plugins/hypervisors/simulator/src/com/cloud/agent/manager/SimulatorManagerImpl.java
View File

@ -29,6 +29,8 @@ import javax.naming.ConfigurationException;
import com.cloud.agent.api.routing.SetMonitorServiceCommand;
import com.cloud.api.commands.ConfigureSimulatorHAProviderState;
import com.cloud.api.commands.ListSimulatorHAStateTransitions;
import org.apache.log4j.Logger;
import org.springframework.stereotype.Component;
import org.apache.cloudstack.storage.command.DeleteCommand;
@ -195,6 +197,8 @@ public class SimulatorManagerImpl extends ManagerBase implements SimulatorManage
cmdList.add(ConfigureSimulatorCmd.class);
cmdList.add(QuerySimulatorMockCmd.class);
cmdList.add(CleanupSimulatorMockCmd.class);
cmdList.add(ConfigureSimulatorHAProviderState.class);
cmdList.add(ListSimulatorHAStateTransitions.class);
return cmdList;
}

120
plugins/hypervisors/simulator/src/com/cloud/api/commands/ConfigureSimulatorHAProviderState.java Normal file
View File

@ -0,0 +1,120 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package com.cloud.api.commands;
import com.cloud.exception.ConcurrentOperationException;
import com.cloud.exception.InsufficientCapacityException;
import com.cloud.exception.NetworkRuleConflictException;
import com.cloud.exception.ResourceAllocationException;
import com.cloud.exception.ResourceUnavailableException;
import com.cloud.host.Host;
import org.apache.cloudstack.acl.RoleType;
import org.apache.cloudstack.api.APICommand;
import org.apache.cloudstack.api.ApiArgValidator;
import org.apache.cloudstack.api.ApiConstants;
import org.apache.cloudstack.api.ApiErrorCode;
import org.apache.cloudstack.api.BaseCmd;
import org.apache.cloudstack.api.Parameter;
import org.apache.cloudstack.api.ServerApiException;
import org.apache.cloudstack.api.response.HostResponse;
import org.apache.cloudstack.api.response.SuccessResponse;
import org.apache.cloudstack.context.CallContext;
import org.apache.cloudstack.ha.HAManager;
import org.apache.cloudstack.ha.SimulatorHAProvider;
import org.apache.cloudstack.ha.SimulatorHAState;
import javax.inject.Inject;
@APICommand(name = ConfigureSimulatorHAProviderState.APINAME,
description="configures simulator HA provider state for a host for probing and testing",
responseObject=SuccessResponse.class,
since = "4.11", authorized = {RoleType.Admin})
public final class ConfigureSimulatorHAProviderState extends BaseCmd {
public static final String APINAME = "configureSimulatorHAProviderState";
@Inject
private HAManager haManager;
/////////////////////////////////////////////////////
//////////////// API parameters /////////////////////
/////////////////////////////////////////////////////
@Parameter(name = ApiConstants.HOST_ID, type = BaseCmd.CommandType.UUID, entityType = HostResponse.class,
description = "List by host ID", required = true, validations = {ApiArgValidator.PositiveNumber})
private Long hostId;
@Parameter(name = ApiConstants.HEALTH, type = CommandType.BOOLEAN,
description = "Set true is haprovider for simulator host should be healthy",
required = true)
private Boolean healthy;
@Parameter(name = ApiConstants.ACTIVITY, type = CommandType.BOOLEAN,
description = "Set true is haprovider for simulator host should have activity",
required = true)
private Boolean activity;
@Parameter(name = ApiConstants.RECOVER, type = CommandType.BOOLEAN,
description = "Set true is haprovider for simulator host should be be recoverable",
required = true)
private Boolean recovery;
@Parameter(name = ApiConstants.FENCE, type = CommandType.BOOLEAN,
description = "Set true is haprovider for simulator host should be be fence-able",
required = true)
private Boolean fenceable;
/////////////////////////////////////////////////////
/////////////////// Accessors ///////////////////////
/////////////////////////////////////////////////////
public Long getHostId() {
return hostId;
}
/////////////////////////////////////////////////////
/////////////// API Implementation///////////////////
/////////////////////////////////////////////////////
@Override
public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException {
final Host host = _resourceService.getHost(getHostId());
if (host == null) {
throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Unable to find host by ID: " + getHostId());
}
final SimulatorHAState haState = new SimulatorHAState(healthy, activity, recovery, fenceable);
final SimulatorHAProvider simulatorHAProvider = (SimulatorHAProvider) haManager.getHAProvider(SimulatorHAProvider.class.getSimpleName().toLowerCase());
if (simulatorHAProvider != null) {
simulatorHAProvider.setHAStateForHost(host.getId(), haState);
}
final SuccessResponse response = new SuccessResponse();
response.setSuccess(simulatorHAProvider != null);
response.setResponseName(getCommandName());
response.setObjectName("simulatorhaprovider");
setResponseObject(response);
}
@Override
public String getCommandName() {
return APINAME.toLowerCase() + BaseCmd.RESPONSE_SUFFIX;
}
@Override
public long getEntityOwnerId() {
return CallContext.current().getCallingAccountId();
}
}

104
plugins/hypervisors/simulator/src/com/cloud/api/commands/ListSimulatorHAStateTransitions.java Normal file
View File

@ -0,0 +1,104 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package com.cloud.api.commands;
import com.cloud.api.response.SimulatorHAStateResponse;
import com.cloud.exception.ConcurrentOperationException;
import com.cloud.exception.InsufficientCapacityException;
import com.cloud.exception.NetworkRuleConflictException;
import com.cloud.exception.ResourceAllocationException;
import com.cloud.exception.ResourceUnavailableException;
import com.cloud.host.Host;
import org.apache.cloudstack.acl.RoleType;
import org.apache.cloudstack.api.APICommand;
import org.apache.cloudstack.api.ApiArgValidator;
import org.apache.cloudstack.api.ApiConstants;
import org.apache.cloudstack.api.ApiErrorCode;
import org.apache.cloudstack.api.BaseCmd;
import org.apache.cloudstack.api.BaseListCmd;
import org.apache.cloudstack.api.Parameter;
import org.apache.cloudstack.api.ServerApiException;
import org.apache.cloudstack.api.response.HostResponse;
import org.apache.cloudstack.api.response.ListResponse;
import org.apache.cloudstack.context.CallContext;
import org.apache.cloudstack.ha.HAManager;
import org.apache.cloudstack.ha.SimulatorHAProvider;
import javax.inject.Inject;
import java.util.ArrayList;
import java.util.List;
@APICommand(name = ListSimulatorHAStateTransitions.APINAME,
description="list recent simulator HA state transitions for a host for probing and testing",
responseObject=SimulatorHAStateResponse.class,
since = "4.11", authorized = {RoleType.Admin})
public final class ListSimulatorHAStateTransitions extends BaseListCmd {
public static final String APINAME = "listSimulatorHAStateTransitions";
@Inject
private HAManager haManager;
/////////////////////////////////////////////////////
//////////////// API parameters /////////////////////
/////////////////////////////////////////////////////
@Parameter(name = ApiConstants.HOST_ID, type = BaseCmd.CommandType.UUID, entityType = HostResponse.class,
description = "List by host ID", required = true, validations = {ApiArgValidator.PositiveNumber})
private Long hostId;
/////////////////////////////////////////////////////
/////////////////// Accessors ///////////////////////
/////////////////////////////////////////////////////
public Long getHostId() {
return hostId;
}
/////////////////////////////////////////////////////
/////////////// API Implementation///////////////////
/////////////////////////////////////////////////////
@Override
public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException {
final Host host = _resourceService.getHost(getHostId());
if (host == null) {
throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Unable to find host by ID: " + getHostId());
}
final SimulatorHAProvider simulatorHAProvider = (SimulatorHAProvider) haManager.getHAProvider(SimulatorHAProvider.class.getSimpleName().toLowerCase());
List<SimulatorHAStateResponse> recentStates = new ArrayList<>();
if (simulatorHAProvider != null) {
recentStates = simulatorHAProvider.listHAStateTransitions(host.getId());
}
final ListResponse<SimulatorHAStateResponse> response = new ListResponse<>();
response.setResponses(recentStates);
response.setResponseName(getCommandName());
response.setObjectName("simulatorhastatetransition");
setResponseObject(response);
}
@Override
public String getCommandName() {
return APINAME.toLowerCase() + BaseCmd.RESPONSE_SUFFIX;
}
@Override
public long getEntityOwnerId() {
return CallContext.current().getCallingAccountId();
}
}

65
plugins/hypervisors/simulator/src/com/cloud/api/response/SimulatorHAStateResponse.java Normal file
View File

@ -0,0 +1,65 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package com.cloud.api.response;
import com.cloud.serializer.Param;
import com.google.gson.annotations.SerializedName;
import org.apache.cloudstack.api.ApiConstants;
import org.apache.cloudstack.api.BaseResponse;
import org.apache.cloudstack.ha.HAConfig;
public class SimulatorHAStateResponse extends BaseResponse {
@SerializedName(ApiConstants.HA_STATE) @Param(description="the ha state")
private String haState;
@SerializedName("prevhastate") @Param(description="the previous ha state")
private String previousHaState;
@SerializedName("event") @Param(description="the event that caused state transition")
private String haEvent;
@SerializedName("activitycounter") @Param(description="the activity counter")
private Long activityCounter;
@SerializedName("recoverycounter") @Param(description="the recovery counter")
private Long recoveryCounter;
public void setHaState(final HAConfig.HAState haState) {
if (haState != null) {
this.haState = haState.toString().toLowerCase();
}
}
public void setPreviousHaState(final HAConfig.HAState previousHaState) {
if (previousHaState != null) {
this.previousHaState = previousHaState.toString().toLowerCase();
}
}
public void setHaEvent(final HAConfig.Event haEvent) {
this.haEvent = haEvent.toString().toLowerCase();
}
public void setActivityCounter(Long activityCounter) {
this.activityCounter = activityCounter;
}
public void setRecoveryCounter(Long recoveryCounter) {
this.recoveryCounter = recoveryCounter;
}
}

11
plugins/hypervisors/simulator/src/com/cloud/ha/SimulatorInvestigator.java
View File

@ -21,6 +21,7 @@ import java.util.List;
import javax.ejb.Local;
import javax.inject.Inject;
import org.apache.cloudstack.ha.HAManager;
import org.apache.log4j.Logger;
import com.cloud.agent.AgentManager;
@ -50,6 +51,8 @@ public class SimulatorInvestigator extends AdapterBase implements Investigator {
ResourceManager _resourceMgr;
@Inject
MockConfigurationDao _mockConfigDao;
@Inject
private HAManager haManager;
protected SimulatorInvestigator() {
}
@ -60,6 +63,10 @@ public class SimulatorInvestigator extends AdapterBase implements Investigator {
return null;
}
if (haManager.isHAEligible(agent)) {
return haManager.getHostStatus(agent);
}
CheckOnHostCommand cmd = new CheckOnHostCommand(agent);
List<HostVO> neighbors = _resourceMgr.listHostsInClusterByStatus(agent.getClusterId(), Status.Up);
for (HostVO neighbor : neighbors) {
@ -81,6 +88,10 @@ public class SimulatorInvestigator extends AdapterBase implements Investigator {
@Override
public Boolean isVmAlive(VirtualMachine vm, Host host) {
if (haManager.isHAEligible(host)) {
return haManager.isVMAliveOnHost(host);
}
CheckVirtualMachineCommand cmd = new CheckVirtualMachineCommand(vm.getInstanceName());
try {
Answer answer = _agentMgr.send(vm.getHostId(), cmd);

152
plugins/hypervisors/simulator/src/org/apache/cloudstack/ha/SimulatorHAProvider.java Normal file
View File

@ -0,0 +1,152 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.ha;
import com.cloud.api.response.SimulatorHAStateResponse;
import com.cloud.host.Host;
import com.cloud.hypervisor.Hypervisor;
import com.cloud.utils.fsm.StateListener;
import com.cloud.utils.fsm.StateMachine2;
import org.apache.cloudstack.ha.provider.HACheckerException;
import org.apache.cloudstack.ha.provider.HAFenceException;
import org.apache.cloudstack.ha.provider.HAProvider;
import org.apache.cloudstack.ha.provider.HARecoveryException;
import org.apache.cloudstack.ha.provider.host.HAAbstractHostProvider;
import org.joda.time.DateTime;
import javax.inject.Inject;
import java.security.InvalidParameterException;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
public class SimulatorHAProvider extends HAAbstractHostProvider implements HAProvider<Host>, StateListener<HAConfig.HAState, HAConfig.Event, HAConfig> {
@Inject
private HAManager haManager;
private final Map<Long, SimulatorHAState> hostHAStateMap = new ConcurrentHashMap<>();
public SimulatorHAProvider() {
HAConfig.HAState.getStateMachine().registerListener(this);
}
public void setHAStateForHost(final Long hostId, final SimulatorHAState state) {
hostHAStateMap.put(hostId, state);
haManager.purgeHACounter(hostId, HAResource.ResourceType.Host);
}
public List<SimulatorHAStateResponse> listHAStateTransitions(final Long hostId) {
final SimulatorHAState haState = hostHAStateMap.get(hostId);
if (haState == null) {
return Collections.emptyList();
}
return haState.listRecentStateTransitions();
}
@Override
public HAResource.ResourceType resourceType() {
return HAResource.ResourceType.Host;
}
@Override
public HAResource.ResourceSubType resourceSubType() {
return HAResource.ResourceSubType.Simulator;
}
@Override
public boolean isEligible(final Host host) {
final SimulatorHAState haState = hostHAStateMap.get(host.getId());
return !isInMaintenanceMode(host) && !isDisabled(host) && haState != null
&& Hypervisor.HypervisorType.Simulator.equals(host.getHypervisorType());
}
@Override
public boolean isHealthy(final Host host) throws HACheckerException {
final SimulatorHAState haState = hostHAStateMap.get(host.getId());
return haState != null && haState.isHealthy();
}
@Override
public boolean hasActivity(final Host host, final DateTime afterThis) throws HACheckerException {
final SimulatorHAState haState = hostHAStateMap.get(host.getId());
return haState != null && haState.hasActivity();
}
@Override
public boolean recover(final Host host) throws HARecoveryException {
final SimulatorHAState haState = hostHAStateMap.get(host.getId());
return haState != null && haState.canRecover();
}
@Override
public boolean fence(final Host host) throws HAFenceException {
final SimulatorHAState haState = hostHAStateMap.get(host.getId());
return haState != null && haState.canFenced();
}
@Override
public Object getConfigValue(final HAProvider.HAProviderConfig name, final Host host) {
switch (name) {
case HealthCheckTimeout:
return 5L;
case ActivityCheckTimeout:
return 5L;
case RecoveryTimeout:
return 5L;
case FenceTimeout:
return 5L;
case MaxActivityCheckInterval:
return 1L;
case MaxActivityChecks:
return 3L;
case ActivityCheckFailureRatio:
final SimulatorHAState haState = hostHAStateMap.get(host.getId());
return (haState != null && haState.hasActivity()) ? 1.0 : 0.0;
case MaxDegradedWaitTimeout:
return 1L;
case MaxRecoveryAttempts:
return 2L;
case RecoveryWaitTimeout:
return 1L;
default:
throw new InvalidParameterException("Unknown HAProviderConfig " + name.toString());
}
}
@Override
public boolean preStateTransitionEvent(final HAConfig.HAState oldState, final HAConfig.Event event,
final HAConfig.HAState newState, final HAConfig vo, final boolean status, final Object opaque) {
return false;
}
@Override
public boolean postStateTransitionEvent(final StateMachine2.Transition<HAConfig.HAState, HAConfig.Event> transition,
final HAConfig vo, final boolean status, final Object opaque) {
if (vo.getResourceType() != HAResource.ResourceType.Host) {
return false;
}
final SimulatorHAState haState = hostHAStateMap.get(vo.getResourceId());
if (haState == null || !status) {
return false;
}
final HAResourceCounter counter = haManager.getHACounter(vo.getResourceId(), vo.getResourceType());
return haState.addStateTransition(transition.getToState(), transition.getCurrentState(), transition.getEvent(), counter);
}
}

89
plugins/hypervisors/simulator/src/org/apache/cloudstack/ha/SimulatorHAState.java Normal file
View File

@ -0,0 +1,89 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.ha;
import com.cloud.api.response.SimulatorHAStateResponse;
import com.google.common.collect.EvictingQueue;
import java.util.ArrayList;
import java.util.List;
import java.util.Queue;
public final class SimulatorHAState {
private boolean healthy;
private boolean activity;
private boolean recover;
private boolean fence;
private Queue<SimulatorHAStateResponse> stateTransitions = EvictingQueue.create(100);
public SimulatorHAState(boolean healthy, boolean activity, boolean recover, boolean fence) {
this.healthy = healthy;
this.activity = activity;
this.recover = recover;
this.fence = fence;
}
public boolean isHealthy() {
return healthy;
}
public void setHealthy(boolean healthy) {
this.healthy = healthy;
}
public boolean hasActivity() {
return activity;
}
public void setActivity(boolean activity) {
this.activity = activity;
}
public boolean canRecover() {
return recover;
}
public void setRecover(boolean recover) {
this.recover = recover;
}
public boolean canFenced() {
return fence;
}
public void setFence(boolean fence) {
this.fence = fence;
}
public boolean addStateTransition(final HAConfig.HAState newHaState, final HAConfig.HAState oldHaState, final HAConfig.Event event, final HAResourceCounter counter) {
final SimulatorHAStateResponse stateResponse = new SimulatorHAStateResponse();
stateResponse.setHaState(newHaState);
stateResponse.setPreviousHaState(oldHaState);
stateResponse.setHaEvent(event);
if (counter != null) {
stateResponse.setActivityCounter(counter.getActivityCheckCounter());
stateResponse.setRecoveryCounter(counter.getRecoveryCounter());
}
stateResponse.setObjectName("hastatetransition");
return stateTransitions.add(stateResponse);
}
public List<SimulatorHAStateResponse> listRecentStateTransitions() {
return new ArrayList<>(stateTransitions);
}
}

1
pom.xml
View File

@ -828,7 +828,6 @@
<exclude>tools/ngui/static/js/lib/*</exclude>
<exclude>**/.checkstyle</exclude>
<exclude>scripts/installer/windows/acs_license.rtf</exclude>
<exclude>test/integration/component/test_host_ha.sh</exclude>
</excludes>
</configuration>
</plugin>

134
scripts/vm/hypervisor/kvm/kvmvmactivity.sh Executable file
View File

@ -0,0 +1,134 @@
#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
help() {
printf "Usage: $0
-i nfs server ip
-p nfs server path
-m mount point
-h host
-u volume uuid list
-t time on ms
-d suspect time\n"
exit 1
}
#set -x
NfsSvrIP=
NfsSvrPath=
MountPoint=
HostIP=
UUIDList=
MSTime=
SuspectTime=
while getopts 'i:p:m:u:t:h:d:' OPTION
do
case $OPTION in
i)
NfsSvrIP="$OPTARG"
;;
p)
NfsSvrPath="$OPTARG"
;;
m)
MountPoint="$OPTARG"
;;
h)
HostIP="$OPTARG"
;;
u)
UUIDList="$OPTARG"
;;
t)
MSTime="$OPTARG"
;;
d)
SuspectTime="$OPTARG"
;;
*)
help
;;
esac
done
if [ -z "$NfsSvrIP" ]
then
exit 2
fi
if [ -z "$SuspectTime" ]
then
exit 2
fi
hbFile="$MountPoint/KVMHA/hb-$HostIP"
acFile="$MountPoint/KVMHA/ac-$HostIP"
# First check: heartbeat file
now=$(date +%s)
hb=$(cat $hbFile)
diff=$(expr $now - $hb)
if [ $diff -lt 61 ]
then
echo "=====> ALIVE <====="
exit 0
fi
if [ -z "$UUIDList" ]
then
echo "=====> DEAD <======"
exit 0
fi
# Second check: disk activity check
cd $MountPoint
latestUpdateTime=$(stat -c %Y $(echo $UUIDList | sed 's/,/ /g') | sort -nr | head -1)
if [ ! -f $acFile ]; then
echo "$SuspectTime:$latestUpdateTime:$MSTime" > $acFile
if [[ $latestUpdateTime -gt $SuspectTime ]]; then
echo "=====> ALIVE <====="
else
echo "=====> DEAD <======"
fi
else
acTime=$(cat $acFile)
arrTime=(${acTime//:/ })
lastSuspectTime=${arrTime[0]}
lastUpdateTime=${arrTime[1]}
echo "$SuspectTime:$latestUpdateTime:$MSTime" > $acFile
if [[ $lastSuspectTime -ne $SuspectTime ]]; then
if [[ $latestUpdateTime -gt $SuspectTime ]]; then
echo "=====> ALIVE <====="
else
echo "=====> DEAD <======"
fi
else
if [[ $latestUpdateTime -gt $lastUpdateTime ]]; then
echo "=====> ALIVE <====="
else
echo "=====> DEAD <======"
fi
fi
fi
exit 0

9
server/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml
View File

@ -70,6 +70,15 @@
value="#{resourceDiscoverersRegistry.registered}" />
</bean>
<!-- the new background poll manager -->
<bean id="bgPollManager" class="org.apache.cloudstack.poll.BackgroundPollManagerImpl">
</bean>
<!-- the new HA manager -->
<bean id="haManagerImpl" class="org.apache.cloudstack.ha.HAManagerImpl">
<property name="haProviders" value="#{haProvidersRegistry.registered}" />
</bean>
<bean id="highAvailabilityManagerExtImpl" class="com.cloud.ha.HighAvailabilityManagerExtImpl">
<property name="investigators" value="#{haInvestigatorsRegistry.registered}" />
<property name="fenceBuilders" value="#{haFenceBuildersRegistry.registered}" />

3
server/src/com/cloud/alert/AlertManagerImpl.java
View File

@ -760,7 +760,8 @@ public class AlertManagerImpl extends ManagerBase implements AlertManager, Confi
(alertType != AlertManager.AlertType.ALERT_TYPE_STORAGE_MISC) &&
(alertType != AlertManager.AlertType.ALERT_TYPE_MANAGMENT_NODE) &&
(alertType != AlertManager.AlertType.ALERT_TYPE_RESOURCE_LIMIT_EXCEEDED) &&
(alertType != AlertManager.AlertType.ALERT_TYPE_OOBM_AUTH_ERROR)) {
(alertType != AlertManager.AlertType.ALERT_TYPE_OOBM_AUTH_ERROR) &&
(alertType != AlertManager.AlertType.ALERT_TYPE_HA_ACTION)) {
alert = _alertDao.getLastAlert(alertType.getType(), dataCenterId, podId, clusterId);
}

5
server/src/com/cloud/api/query/dao/HostJoinDaoImpl.java
View File

@ -28,6 +28,8 @@ import javax.ejb.Local;
import javax.inject.Inject;
import com.cloud.host.dao.HostDetailsDao;
import org.apache.cloudstack.ha.HAResource;
import org.apache.cloudstack.ha.dao.HAConfigDao;
import org.apache.cloudstack.outofbandmanagement.dao.OutOfBandManagementDao;
import org.apache.log4j.Logger;
import org.springframework.stereotype.Component;
@ -61,6 +63,8 @@ public class HostJoinDaoImpl extends GenericDaoBase<HostJoinVO, Long> implements
@Inject
private HostDetailsDao hostDetailsDao;
@Inject
private HAConfigDao haConfigDao;
@Inject
private OutOfBandManagementDao outOfBandManagementDao;
private final SearchBuilder<HostJoinVO> hostSearch;
@ -224,6 +228,7 @@ public class HostJoinDaoImpl extends GenericDaoBase<HostJoinVO, Long> implements
}
}
hostResponse.setHostHAResponse(haConfigDao.findHAResource(host.getId(), HAResource.ResourceType.Host));
hostResponse.setOutOfBandManagementResponse(outOfBandManagementDao.findByHost(host.getId()));
hostResponse.setResourceState(host.getResourceState().toString());

22
server/src/com/cloud/api/query/vo/HostJoinVO.java
View File

@ -36,6 +36,7 @@ import com.cloud.hypervisor.Hypervisor.HypervisorType;
import com.cloud.org.Cluster;
import com.cloud.resource.ResourceState;
import com.cloud.utils.db.GenericDao;
import org.apache.cloudstack.ha.HAConfig;
import org.apache.cloudstack.outofbandmanagement.OutOfBandManagement;
/**
@ -99,6 +100,15 @@ public class HostJoinVO extends BaseViewVO implements InternalIdentity, Identity
@Enumerated(value = EnumType.STRING)
private OutOfBandManagement.PowerState outOfBandManagementPowerState;
@Column(name = "ha_enabled")
private boolean hostHAEnabled = false;
@Column(name = "ha_state")
private HAConfig.HAState hostHAState;
@Column(name = "ha_provider")
private String hostHAProvider;
@Column(name = "resource_state")
@Enumerated(value = EnumType.STRING)
private ResourceState resourceState;
@ -260,6 +270,18 @@ public class HostJoinVO extends BaseViewVO implements InternalIdentity, Identity
return outOfBandManagementPowerState;
}
public boolean isHostHAEnabled() {
return hostHAEnabled;
}
public HAConfig.HAState getHostHAState() {
return hostHAState;
}
public String getHostHAProvider() {
return hostHAProvider;
}
public ResourceState getResourceState() {
return resourceState;
}

46
server/src/com/cloud/server/StatsCollector.java
View File

@ -18,7 +18,6 @@ package com.cloud.server;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
@ -32,11 +31,6 @@ import java.util.concurrent.TimeUnit;
import javax.inject.Inject;
import org.apache.cloudstack.outofbandmanagement.OutOfBandManagement;
import org.apache.cloudstack.outofbandmanagement.OutOfBandManagementService;
import org.apache.cloudstack.outofbandmanagement.OutOfBandManagementVO;
import org.apache.cloudstack.outofbandmanagement.dao.OutOfBandManagementDao;
import org.apache.cloudstack.utils.identity.ManagementServerNode;
import org.apache.cloudstack.utils.usage.UsageUtils;
import org.apache.log4j.Logger;
import org.springframework.stereotype.Component;
@ -139,8 +133,6 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc
@Inject
private HostDao _hostDao;
@Inject
private OutOfBandManagementDao outOfBandManagementDao;
@Inject
private UserVmDao _userVmDao;
@Inject
private VolumeDao _volsDao;
@ -157,8 +149,6 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc
@Inject
private ResourceManager _resourceMgr;
@Inject
private OutOfBandManagementService outOfBandManagementService;
@Inject
private ConfigurationDao _configDao;
@Inject
private EndPointSelector _epSelector;
@ -198,7 +188,6 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc
private ConcurrentHashMap<Long, StorageStats> _storagePoolStats = new ConcurrentHashMap<Long, StorageStats>();
long hostStatsInterval = -1L;
long hostOutOfBandManagementStatsInterval = -1L;
long hostAndVmStatsInterval = -1L;
long storageStatsInterval = -1L;
long volumeStatsInterval = -1L;
@ -237,7 +226,6 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc
private void init(Map<String, String> configs) {
_executor = Executors.newScheduledThreadPool(6, new NamedThreadFactory("StatsCollector"));
hostOutOfBandManagementStatsInterval = OutOfBandManagementService.SyncThreadInterval.value();
hostStatsInterval = NumbersUtil.parseLong(configs.get("host.stats.interval"), 60000L);
hostAndVmStatsInterval = NumbersUtil.parseLong(configs.get("vm.stats.interval"), 60000L);
storageStatsInterval = NumbersUtil.parseLong(configs.get("storage.stats.interval"), 60000L);
@ -249,10 +237,6 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc
_executor.scheduleWithFixedDelay(new HostCollector(), 15000L, hostStatsInterval, TimeUnit.MILLISECONDS);
}
if (hostOutOfBandManagementStatsInterval > 0) {
_executor.scheduleWithFixedDelay(new HostOutOfBandManagementStatsCollector(), 15000L, hostOutOfBandManagementStatsInterval, TimeUnit.MILLISECONDS);
}
if (hostAndVmStatsInterval > 0) {
_executor.scheduleWithFixedDelay(new VmStatsCollector(), 15000L, hostAndVmStatsInterval, TimeUnit.MILLISECONDS);
}
@ -371,36 +355,6 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc
}
}
class HostOutOfBandManagementStatsCollector extends ManagedContextRunnable {
@Override
protected void runInContext() {
try {
s_logger.debug("HostOutOfBandManagementStatsCollector is running...");
List<OutOfBandManagementVO> outOfBandManagementHosts = outOfBandManagementDao.findAllByManagementServer(ManagementServerNode.getManagementServerId());
if (outOfBandManagementHosts == null) {
return;
}
for (OutOfBandManagement outOfBandManagementHost : outOfBandManagementHosts) {
Host host = _hostDao.findById(outOfBandManagementHost.getHostId());
if (host == null) {
continue;
}
if (outOfBandManagementService.isOutOfBandManagementEnabled(host)) {
outOfBandManagementService.submitBackgroundPowerSyncTask(host);
} else if (outOfBandManagementHost.getPowerState() != OutOfBandManagement.PowerState.Disabled) {
if (outOfBandManagementService.transitionPowerStateToDisabled(Collections.singletonList(host))) {
if (s_logger.isDebugEnabled()) {
s_logger.debug("Out-of-band management was disabled in zone/cluster/host, disabled power state for host id:" + host.getId());
}
}
}
}
} catch (Throwable t) {
s_logger.error("Error trying to retrieve host out-of-band management stats", t);
}
}
}
class VmStatsCollector extends ManagedContextRunnable {
@Override
protected void runInContext() {

3
server/src/com/cloud/storage/StorageManagerImpl.java
View File

@ -1006,7 +1006,6 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
Command[] cmdArray = cmds.toCommands();
for (Command cmd : cmdArray) {
long targetHostId = _hvGuruMgr.getGuruProcessedCommandTargetHost(hostId, cmd);
answers.add(_agentMgr.send(targetHostId, cmd));
}
return new Pair<Long, Answer[]>(hostId, answers.toArray(new Answer[answers.size()]));
@ -2075,8 +2074,6 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
" for template id " +templateOnImageStore.getTemplateId(), th);
}
}
}
// get bytesReadRate from service_offering, disk_offering and vm.disk.throttling.bytes_read_rate

76
server/src/org/apache/cloudstack/ha/HAManager.java Normal file
View File

@ -0,0 +1,76 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.ha;
import com.cloud.host.Host;
import com.cloud.host.Status;
import org.apache.cloudstack.framework.config.ConfigKey;
import org.apache.cloudstack.ha.provider.HAProvider;
public interface HAManager extends HAConfigManager {
ConfigKey<Integer> MaxConcurrentHealthCheckOperations = new ConfigKey<>("Advanced", Integer.class,
"ha.max.concurrent.health.check.operations",
"50",
"The number of concurrent health check operations per management server. This setting determines the size of the thread pool consuming the HEALTH CHECK queue.", true);
ConfigKey<Integer> MaxPendingHealthCheckOperations = new ConfigKey<>("Advanced", Integer.class,
"ha.max.pending.health.check.operations",
"5000",
"The number of pending health check operations per management server. This setting determines the size of the HEALTH CHECK queue.", true);
ConfigKey<Integer> MaxConcurrentActivityCheckOperations = new ConfigKey<>("Advanced", Integer.class,
"ha.max.concurrent.activity.check.operations",
"25",
"The number of concurrent activity check operations per management server. This setting determines the size of the thread pool consuming the ACTIVITY CHECK queue.",
true);
ConfigKey<Integer> MaxPendingActivityCheckOperations = new ConfigKey<>("Advanced", Integer.class,
"ha.max.pending.activity.check.operations",
"2500",
"The number of pending activity check operations per management server. This setting determines the size of the size of the ACTIVITY CHECK queue.", true);
ConfigKey<Integer> MaxConcurrentRecoveryOperations = new ConfigKey<>("Advanced", Integer.class,
"ha.max.concurrent.recovery.operations",
"25",
"The number of concurrent recovery operations per management server.", true);
ConfigKey<Integer> MaxPendingRecoveryOperations = new ConfigKey<>("Advanced", Integer.class,
"ha.max.pending.recovery.operations",
"2500",
"The number of pending recovery operations per management server. This setting determines the size of the size of the RECOVERY queue.", true);
ConfigKey<Integer> MaxConcurrentFenceOperations = new ConfigKey<>("Advanced", Integer.class,
"ha.max.concurrent.fence.operations",
"25",
"The number of concurrent fence operations per management server.", true);
ConfigKey<Integer> MaxPendingFenceOperations = new ConfigKey<>("Advanced", Integer.class,
"ha.max.pending.fence.operations",
"2500",
"The number of pending fence operations per management server. This setting determines the size of the size of the FENCE queue.", true);
boolean transitionHAState(final HAConfig.Event event, final HAConfig haConfig);
HAProvider getHAProvider(final String name);
HAResourceCounter getHACounter(final Long resourceId, final HAResource.ResourceType resourceType);
void purgeHACounter(final Long resourceId, final HAResource.ResourceType resourceType);
boolean isHAEligible(final HAResource resource);
Boolean isVMAliveOnHost(final Host host);
Status getHostStatus(final Host host);
}

743
server/src/org/apache/cloudstack/ha/HAManagerImpl.java Normal file
View File

@ -0,0 +1,743 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.ha;
import com.cloud.cluster.ClusterManagerListener;
import com.cloud.cluster.ManagementServerHost;
import com.cloud.dc.ClusterDetailsDao;
import com.cloud.dc.ClusterDetailsVO;
import com.cloud.dc.DataCenter;
import com.cloud.dc.DataCenterDetailVO;
import com.cloud.dc.dao.DataCenterDetailsDao;
import com.cloud.domain.Domain;
import com.cloud.event.ActionEvent;
import com.cloud.event.ActionEventUtils;
import com.cloud.event.EventTypes;
import com.cloud.host.Host;
import com.cloud.host.Status;
import com.cloud.host.dao.HostDao;
import com.cloud.org.Cluster;
import com.cloud.utils.component.ComponentContext;
import com.cloud.utils.component.ManagerBase;
import com.cloud.utils.component.PluggableService;
import com.cloud.utils.db.Transaction;
import com.cloud.utils.db.TransactionCallback;
import com.cloud.utils.db.TransactionStatus;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.fsm.NoTransitionException;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import org.apache.cloudstack.api.ApiErrorCode;
import org.apache.cloudstack.api.ServerApiException;
import org.apache.cloudstack.api.command.admin.ha.ConfigureHAForHostCmd;
import org.apache.cloudstack.api.command.admin.ha.DisableHAForClusterCmd;
import org.apache.cloudstack.api.command.admin.ha.DisableHAForHostCmd;
import org.apache.cloudstack.api.command.admin.ha.DisableHAForZoneCmd;
import org.apache.cloudstack.api.command.admin.ha.EnableHAForClusterCmd;
import org.apache.cloudstack.api.command.admin.ha.EnableHAForHostCmd;
import org.apache.cloudstack.api.command.admin.ha.EnableHAForZoneCmd;
import org.apache.cloudstack.api.command.admin.ha.ListHostHAProvidersCmd;
import org.apache.cloudstack.api.command.admin.ha.ListHostHAResourcesCmd;
import org.apache.cloudstack.context.CallContext;
import org.apache.cloudstack.framework.config.ConfigKey;
import org.apache.cloudstack.framework.config.Configurable;
import org.apache.cloudstack.ha.dao.HAConfigDao;
import org.apache.cloudstack.ha.provider.HAProvider;
import org.apache.cloudstack.ha.provider.HAProvider.HAProviderConfig;
import org.apache.cloudstack.ha.task.ActivityCheckTask;
import org.apache.cloudstack.ha.task.FenceTask;
import org.apache.cloudstack.ha.task.HealthCheckTask;
import org.apache.cloudstack.ha.task.RecoveryTask;
import org.apache.cloudstack.kernel.Partition;
import org.apache.cloudstack.managed.context.ManagedContextRunnable;
import org.apache.cloudstack.poll.BackgroundPollManager;
import org.apache.cloudstack.poll.BackgroundPollTask;
import org.apache.cloudstack.utils.identity.ManagementServerNode;
import org.apache.log4j.Logger;
import javax.inject.Inject;
import javax.naming.ConfigurationException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
public final class HAManagerImpl extends ManagerBase implements HAManager, ClusterManagerListener, PluggableService, Configurable {
public static final Logger LOG = Logger.getLogger(HAManagerImpl.class);
@Inject
private HAConfigDao haConfigDao;
@Inject
private HostDao hostDao;
@Inject
private ClusterDetailsDao clusterDetailsDao;
@Inject
private DataCenterDetailsDao dataCenterDetailsDao;
@Inject
private BackgroundPollManager pollManager;
private List<HAProvider<HAResource>> haProviders;
private Map<String, HAProvider<HAResource>> haProviderMap = new HashMap<>();
private static ExecutorService healthCheckExecutor;
private static ExecutorService activityCheckExecutor;
private static ExecutorService recoveryExecutor;
private static ExecutorService fenceExecutor;
private static final String HA_ENABLED_DETAIL = "resourceHAEnabled";
//////////////////////////////////////////////////////
//////////////// HA Manager methods //////////////////
//////////////////////////////////////////////////////
public Map<String, HAResourceCounter> haCounterMap = new ConcurrentHashMap<>();
public HAProvider<HAResource> getHAProvider(final String name) {
return haProviderMap.get(name);
}
private String resourceCounterKey(final Long resourceId, final HAResource.ResourceType resourceType) {
return resourceId.toString() + resourceType.toString();
}
public synchronized HAResourceCounter getHACounter(final Long resourceId, final HAResource.ResourceType resourceType) {
final String key = resourceCounterKey(resourceId, resourceType);
if (!haCounterMap.containsKey(key)) {
haCounterMap.put(key, new HAResourceCounter());
}
return haCounterMap.get(key);
}
public synchronized void purgeHACounter(final Long resourceId, final HAResource.ResourceType resourceType) {
final String key = resourceCounterKey(resourceId, resourceType);
if (haCounterMap.containsKey(key)) {
haCounterMap.remove(key);
}
}
public boolean transitionHAState(final HAConfig.Event event, final HAConfig haConfig) {
if (event == null || haConfig == null) {
return false;
}
final HAConfig.HAState currentHAState = haConfig.getState();
try {
final HAConfig.HAState nextState = HAConfig.HAState.getStateMachine().getNextState(currentHAState, event);
boolean result = HAConfig.HAState.getStateMachine().transitTo(haConfig, event, null, haConfigDao);
if (result) {
final String message = String.format("Transitioned host HA state from:%s to:%s due to event:%s for the host id:%d",
currentHAState, nextState, event, haConfig.getResourceId());
LOG.debug(message);
if (nextState == HAConfig.HAState.Recovering || nextState == HAConfig.HAState.Fencing || nextState == HAConfig.HAState.Fenced) {
ActionEventUtils.onActionEvent(CallContext.current().getCallingUserId(), CallContext.current().getCallingAccountId(),
Domain.ROOT_DOMAIN, EventTypes.EVENT_HA_STATE_TRANSITION, message);
}
}
return result;
} catch (NoTransitionException e) {
if (LOG.isTraceEnabled()) {
LOG.trace("Unable to find next HA state for current HA state: " + currentHAState + " for event: " + event + " for host" + haConfig.getResourceId());
}
}
return false;
}
private boolean transitionResourceStateToDisabled(final Partition partition) {
List<? extends HAResource> resources;
if (partition.partitionType() == Partition.PartitionType.Cluster) {
resources = hostDao.findByClusterId(partition.getId());
} else if (partition.partitionType() == Partition.PartitionType.Zone) {
resources = hostDao.findByDataCenterId(partition.getId());
} else {
return true;
}
boolean result = true;
for (final HAResource resource: resources) {
result = result && transitionHAState(HAConfig.Event.Disabled,
haConfigDao.findHAResource(resource.getId(), resource.resourceType()));
}
return result;
}
private boolean checkHAOwnership(final HAConfig haConfig) {
// Skip for resources not owned by this mgmt server
return !(haConfig.getManagementServerId() != null
&& haConfig.getManagementServerId() != ManagementServerNode.getManagementServerId());
}
private HAResource validateAndFindHAResource(final HAConfig haConfig) {
HAResource resource = null;
if (haConfig.getResourceType() == HAResource.ResourceType.Host) {
final Host host = hostDao.findById(haConfig.getResourceId());
if (host != null && host.getRemoved() != null) {
return null;
}
resource = host;
if (resource == null && haConfig.getState() != HAConfig.HAState.Disabled) {
disableHA(haConfig.getResourceId(), haConfig.getResourceType());
return null;
}
}
if (!haConfig.isEnabled() || !isHAEnabledForZone(resource) || !isHAEnabledForCluster(resource)) {
if (haConfig.getState() != HAConfig.HAState.Disabled) {
if (transitionHAState(HAConfig.Event.Disabled, haConfig) ) {
purgeHACounter(haConfig.getResourceId(), haConfig.getResourceType());
}
}
return null;
} else if (haConfig.getState() == HAConfig.HAState.Disabled) {
transitionHAState(HAConfig.Event.Enabled, haConfig);
}
return resource;
}
private HAProvider<HAResource> validateAndFindHAProvider(final HAConfig haConfig, final HAResource resource) {
final HAProvider<HAResource> haProvider = haProviderMap.get(haConfig.getHaProvider());
if (haProvider != null && !haProvider.isEligible(resource)) {
if (haConfig.getState() != HAConfig.HAState.Ineligible) {
transitionHAState(HAConfig.Event.Ineligible, haConfig);
}
return null;
} else if (haConfig.getState() == HAConfig.HAState.Ineligible) {
transitionHAState(HAConfig.Event.Eligible, haConfig);
}
return haProvider;
}
public boolean isHAEnabledForZone(final HAResource resource) {
if (resource == null || resource.getDataCenterId() < 1L) {
return true;
}
final DataCenterDetailVO zoneDetails = dataCenterDetailsDao.findDetail(resource.getDataCenterId(), HA_ENABLED_DETAIL);
return zoneDetails == null || Strings.isNullOrEmpty(zoneDetails.getValue()) || Boolean.valueOf(zoneDetails.getValue());
}
private boolean isHAEnabledForCluster(final HAResource resource) {
if (resource == null || resource.getClusterId() == null) {
return true;
}
final ClusterDetailsVO clusterDetails = clusterDetailsDao.findDetail(resource.getClusterId(), HA_ENABLED_DETAIL);
return clusterDetails == null || Strings.isNullOrEmpty(clusterDetails.getValue()) || Boolean.valueOf(clusterDetails.getValue());
}
private boolean isHAEligibleForResource(final HAResource resource) {
if (resource == null || resource.getId() < 1L) {
return false;
}
HAResource.ResourceType resourceType = null;
if (resource instanceof Host) {
resourceType = HAResource.ResourceType.Host;
}
if (resourceType == null) {
return false;
}
final HAConfig haConfig = haConfigDao.findHAResource(resource.getId(), resourceType);
return haConfig != null && haConfig.isEnabled()
&& haConfig.getState() != HAConfig.HAState.Disabled
&& haConfig.getState() != HAConfig.HAState.Ineligible;
}
public boolean isHAEligible(final HAResource resource) {
return resource != null && isHAEnabledForZone(resource)
&& isHAEnabledForCluster(resource)
&& isHAEligibleForResource(resource);
}
public void validateHAProviderConfigForResource(final Long resourceId, final HAResource.ResourceType resourceType, final HAProvider<HAResource> haProvider) {
if (HAResource.ResourceType.Host.equals(resourceType)) {
final Host host = hostDao.findById(resourceId);
if (host.getHypervisorType() == null || haProvider.resourceSubType() == null || !host.getHypervisorType().toString().equals(haProvider.resourceSubType().toString())) {
throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Incompatible haprovider provided for the resource of hypervisor type:" + host.getHypervisorType());
}
}
}
////////////////////////////////////////////////////////////////////
//////////////// HA Investigator wrapper for Old HA ////////////////
////////////////////////////////////////////////////////////////////
public Boolean isVMAliveOnHost(final Host host) {
final HAConfig haConfig = haConfigDao.findHAResource(host.getId(), HAResource.ResourceType.Host);
if (haConfig != null) {
if (haConfig.getState() == HAConfig.HAState.Fenced) {
if (LOG.isDebugEnabled()){
LOG.debug("HA: Host is fenced " + host.getId());
}
return false;
}
if (LOG.isDebugEnabled()){
LOG.debug("HA: HOST is alive " + host.getId());
}
return true;
}
return null;
}
public Status getHostStatus(final Host host) {
final HAConfig haConfig = haConfigDao.findHAResource(host.getId(), HAResource.ResourceType.Host);
if (haConfig != null) {
if (haConfig.getState() == HAConfig.HAState.Fenced) {
if (LOG.isDebugEnabled()){
LOG.debug("HA: Agent is available/suspect/checking Up " + host.getId());
}
return Status.Down;
} else if (haConfig.getState() == HAConfig.HAState.Degraded || haConfig.getState() == HAConfig.HAState.Recovering || haConfig.getState() == HAConfig.HAState.Recovered || haConfig.getState() == HAConfig.HAState.Fencing) {
if (LOG.isDebugEnabled()){
LOG.debug("HA: Agent is disconnected " + host.getId());
}
return Status.Disconnected;
}
return Status.Up;
}
return Status.Unknown;
}
//////////////////////////////////////////////////////
//////////////// HA API handlers /////////////////////
//////////////////////////////////////////////////////
private boolean configureHA(final Long resourceId, final HAResource.ResourceType resourceType, final Boolean enable, final String haProvider) {
return Transaction.execute(new TransactionCallback<Boolean>() {
@Override
public Boolean doInTransaction(TransactionStatus status) {
HAConfigVO haConfig = (HAConfigVO) haConfigDao.findHAResource(resourceId, resourceType);
if (haConfig == null) {
haConfig = new HAConfigVO();
if (haProvider != null) {
haConfig.setHaProvider(haProvider);
}
if (enable != null) {
haConfig.setEnabled(enable);
haConfig.setManagementServerId(ManagementServerNode.getManagementServerId());
}
haConfig.setResourceId(resourceId);
haConfig.setResourceType(resourceType);
if (Strings.isNullOrEmpty(haConfig.getHaProvider())) {
throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "HAProvider is not provided for the resource, failing configuration.");
}
if (haConfigDao.persist(haConfig) != null) {
return true;
}
} else {
if (enable != null) {
haConfig.setEnabled(enable);
}
if (haProvider != null) {
haConfig.setHaProvider(haProvider);
}
if (Strings.isNullOrEmpty(haConfig.getHaProvider())) {
throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "HAProvider is not provided for the resource, failing configuration.");
}
return haConfigDao.update(haConfig.getId(), haConfig);
}
return false;
}
});
}
@Override
@ActionEvent(eventType = EventTypes.EVENT_HA_RESOURCE_CONFIGURE, eventDescription = "configuring HA for resource")
public boolean configureHA(final Long resourceId, final HAResource.ResourceType resourceType, final String haProvider) {
Preconditions.checkArgument(resourceId != null && resourceId > 0L);
Preconditions.checkArgument(resourceType != null);
Preconditions.checkArgument(!Strings.isNullOrEmpty(haProvider));
if (!haProviderMap.containsKey(haProvider.toLowerCase())) {
throw new CloudRuntimeException("Given HA provider does not exist.");
}
validateHAProviderConfigForResource(resourceId, resourceType, haProviderMap.get(haProvider.toLowerCase()));
return configureHA(resourceId, resourceType, null, haProvider.toLowerCase());
}
@Override
@ActionEvent(eventType = EventTypes.EVENT_HA_RESOURCE_ENABLE, eventDescription = "enabling HA for resource")
public boolean enableHA(final Long resourceId, final HAResource.ResourceType resourceType) {
Preconditions.checkArgument(resourceId != null && resourceId > 0L);
Preconditions.checkArgument(resourceType != null);
return configureHA(resourceId, resourceType, true, null);
}
@Override
@ActionEvent(eventType = EventTypes.EVENT_HA_RESOURCE_DISABLE, eventDescription = "disabling HA for resource")
public boolean disableHA(final Long resourceId, final HAResource.ResourceType resourceType) {
Preconditions.checkArgument(resourceId != null && resourceId > 0L);
Preconditions.checkArgument(resourceType != null);
boolean result = configureHA(resourceId, resourceType, false, null);
if (result) {
transitionHAState(HAConfig.Event.Disabled, haConfigDao.findHAResource(resourceId, resourceType));
purgeHACounter(resourceId, resourceType);
}
return result;
}
@Override
@ActionEvent(eventType = EventTypes.EVENT_HA_RESOURCE_ENABLE, eventDescription = "enabling HA for a cluster")
public boolean enableHA(final Cluster cluster) {
clusterDetailsDao.persist(cluster.getId(), HA_ENABLED_DETAIL, String.valueOf(true));
return true;
}
@Override
@ActionEvent(eventType = EventTypes.EVENT_HA_RESOURCE_DISABLE, eventDescription = "disabling HA for a cluster")
public boolean disableHA(final Cluster cluster) {
clusterDetailsDao.persist(cluster.getId(), HA_ENABLED_DETAIL, String.valueOf(false));
return transitionResourceStateToDisabled(cluster);
}
@Override
@ActionEvent(eventType = EventTypes.EVENT_HA_RESOURCE_ENABLE, eventDescription = "enabling HA for a zone")
public boolean enableHA(final DataCenter zone) {
dataCenterDetailsDao.persist(zone.getId(), HA_ENABLED_DETAIL, String.valueOf(true));
return true;
}
@Override
@ActionEvent(eventType = EventTypes.EVENT_HA_RESOURCE_DISABLE, eventDescription = "disabling HA for a zone")
public boolean disableHA(final DataCenter zone) {
dataCenterDetailsDao.persist(zone.getId(), HA_ENABLED_DETAIL, String.valueOf(false));
return transitionResourceStateToDisabled(zone);
}
@Override
public List<HAConfig> listHAResources(final Long resourceId, final HAResource.ResourceType resourceType) {
return haConfigDao.listHAResource(resourceId, resourceType);
}
@Override
public List<String> listHAProviders(final HAResource.ResourceType resourceType, final HAResource.ResourceSubType entityType) {
final List<String> haProviderNames = new ArrayList<>();
for (final HAProvider<HAResource> haProvider : haProviders) {
if (haProvider.resourceType().equals(resourceType) && haProvider.resourceSubType().equals(entityType)) {
haProviderNames.add(haProvider.getClass().getSimpleName());
}
}
return haProviderNames;
}
@Override
public List<Class<?>> getCommands() {
List<Class<?>> cmdList = new ArrayList<>();
cmdList.add(ConfigureHAForHostCmd.class);
cmdList.add(EnableHAForHostCmd.class);
cmdList.add(EnableHAForClusterCmd.class);
cmdList.add(EnableHAForZoneCmd.class);
cmdList.add(DisableHAForHostCmd.class);
cmdList.add(DisableHAForClusterCmd.class);
cmdList.add(DisableHAForZoneCmd.class);
cmdList.add(ListHostHAResourcesCmd.class);
cmdList.add(ListHostHAProvidersCmd.class);
return cmdList;
}
//////////////////////////////////////////////////////////////////
//////////////// Clustered Manager Listeners /////////////////////
//////////////////////////////////////////////////////////////////
@Override
public void onManagementNodeJoined(List<? extends ManagementServerHost> nodeList, long selfNodeId) {
}
@Override
public void onManagementNodeLeft(List<? extends ManagementServerHost> nodeList, long selfNodeId) {
}
@Override
public void onManagementNodeIsolated() {
}
///////////////////////////////////////////////////
//////////////// Manager Init /////////////////////
///////////////////////////////////////////////////
@Override
public boolean start() {
haProviderMap.clear();
for (final HAProvider<HAResource> haProvider : haProviders) {
haProviderMap.put(haProvider.getClass().getSimpleName().toLowerCase(), haProvider);
}
return true;
}
@Override
public boolean stop() {
haConfigDao.expireServerOwnership(ManagementServerNode.getManagementServerId());
return true;
}
@Override
public boolean configure(final String name, final Map<String, Object> params) throws ConfigurationException {
// Health Check
final int healthCheckWorkers = MaxConcurrentHealthCheckOperations.value();
final int healthCheckQueueSize = MaxPendingHealthCheckOperations.value();
healthCheckExecutor = new ThreadPoolExecutor(healthCheckWorkers, healthCheckWorkers,
0L, TimeUnit.MILLISECONDS,
new ArrayBlockingQueue<Runnable>(healthCheckQueueSize, true), new ThreadPoolExecutor.CallerRunsPolicy());
// Activity Check
final int activityCheckWorkers = MaxConcurrentActivityCheckOperations.value();
final int activityCheckQueueSize = MaxPendingActivityCheckOperations.value();
activityCheckExecutor = new ThreadPoolExecutor(activityCheckWorkers, activityCheckWorkers,
0L, TimeUnit.MILLISECONDS,
new ArrayBlockingQueue<Runnable>(activityCheckQueueSize, true), new ThreadPoolExecutor.CallerRunsPolicy());
// Recovery
final int recoveryOperationWorkers = MaxConcurrentRecoveryOperations.value();
final int recoveryOperationQueueSize = MaxPendingRecoveryOperations.value();
recoveryExecutor = new ThreadPoolExecutor(recoveryOperationWorkers, recoveryOperationWorkers,
0L, TimeUnit.MILLISECONDS,
new ArrayBlockingQueue<Runnable>(recoveryOperationQueueSize, true), new ThreadPoolExecutor.CallerRunsPolicy());
// Fence
final int fenceOperationWorkers = MaxConcurrentFenceOperations.value();
final int fenceOperationQueueSize = MaxPendingFenceOperations.value();
fenceExecutor = new ThreadPoolExecutor(fenceOperationWorkers, fenceOperationWorkers,
0L, TimeUnit.MILLISECONDS,
new ArrayBlockingQueue<Runnable>(fenceOperationQueueSize, true), new ThreadPoolExecutor.CallerRunsPolicy());
pollManager.submitTask(new HealthCheckPollTask());
pollManager.submitTask(new ActivityCheckPollTask());
pollManager.submitTask(new RecoveryPollTask());
pollManager.submitTask(new FencingPollTask());
LOG.debug("HA manager has been configured");
return true;
}
public void setHaProviders(List<HAProvider<HAResource>> haProviders) {
this.haProviders = haProviders;
}
@Override
public String getConfigComponentName() {
return HAManager.class.getSimpleName();
}
@Override
public ConfigKey<?>[] getConfigKeys() {
return new ConfigKey<?>[] {
MaxConcurrentHealthCheckOperations,
MaxPendingHealthCheckOperations,
MaxConcurrentActivityCheckOperations,
MaxPendingActivityCheckOperations,
MaxConcurrentRecoveryOperations,
MaxPendingRecoveryOperations,
MaxConcurrentFenceOperations,
MaxPendingFenceOperations
};
}
/////////////////////////////////////////////////
//////////////// Poll Tasks /////////////////////
/////////////////////////////////////////////////
private final class HealthCheckPollTask extends ManagedContextRunnable implements BackgroundPollTask {
@Override
protected void runInContext() {
try {
if (LOG.isTraceEnabled()) {
LOG.trace("HA health check task is running...");
}
final List<HAConfig> haConfigList = new ArrayList<HAConfig>(haConfigDao.listAll());
for (final HAConfig haConfig : haConfigList) {
if (!checkHAOwnership(haConfig)) {
continue;
}
final HAResource resource = validateAndFindHAResource(haConfig);
if (resource == null) {
continue;
}
final HAProvider<HAResource> haProvider = validateAndFindHAProvider(haConfig, resource);
if (haProvider == null) {
continue;
}
final HAResourceCounter counter = getHACounter(haConfig.getResourceId(), haConfig.getResourceType());
if (haConfig.getState() == HAConfig.HAState.Suspect) {
if (counter.canPerformActivityCheck((Long)(haProvider.getConfigValue(HAProviderConfig.MaxActivityCheckInterval, resource)))) {
transitionHAState(HAConfig.Event.PerformActivityCheck, haConfig);
}
}
if (haConfig.getState() == HAConfig.HAState.Degraded) {
if (counter.canRecheckActivity((Long)(haProvider.getConfigValue(HAProviderConfig.MaxDegradedWaitTimeout, resource)))) {
transitionHAState(HAConfig.Event.PeriodicRecheckResourceActivity, haConfig);
}
}
switch (haConfig.getState()) {
case Available:
case Suspect:
case Degraded:
case Fenced:
final HealthCheckTask task = ComponentContext.inject(new HealthCheckTask(resource, haProvider, haConfig,
HAProviderConfig.HealthCheckTimeout, healthCheckExecutor));
healthCheckExecutor.submit(task);
break;
default:
break;
}
}
} catch (Throwable t) {
LOG.error("Error trying to perform health checks in HA manager", t);
}
}
}
private final class ActivityCheckPollTask extends ManagedContextRunnable implements BackgroundPollTask {
@Override
protected void runInContext() {
try {
if (LOG.isTraceEnabled()) {
LOG.trace("HA activity check task is running...");
}
final List<HAConfig> haConfigList = new ArrayList<HAConfig>(haConfigDao.listAll());
for (final HAConfig haConfig : haConfigList) {
if (!checkHAOwnership(haConfig)) {
continue;
}
final HAResource resource = validateAndFindHAResource(haConfig);
if (resource == null) {
continue;
}
final HAProvider<HAResource> haProvider = validateAndFindHAProvider(haConfig, resource);
if (haProvider == null) {
continue;
}
if (haConfig.getState() == HAConfig.HAState.Checking) {
final HAResourceCounter counter = getHACounter(haConfig.getResourceId(), haConfig.getResourceType());
final ActivityCheckTask job = ComponentContext.inject(new ActivityCheckTask(resource, haProvider, haConfig,
HAProviderConfig.ActivityCheckTimeout, activityCheckExecutor, counter.getSuspectTimeStamp()));
activityCheckExecutor.submit(job);
}
}
} catch (Throwable t) {
LOG.error("Error trying to perform activity checks in HA manager", t);
}
}
}
private final class RecoveryPollTask extends ManagedContextRunnable implements BackgroundPollTask {
@Override
protected void runInContext() {
try {
if (LOG.isTraceEnabled()) {
LOG.trace("HA recovery task is running...");
}
final List<HAConfig> haConfigList = new ArrayList<HAConfig>(haConfigDao.listAll());
for (final HAConfig haConfig : haConfigList) {
if (!checkHAOwnership(haConfig)) {
continue;
}
final HAResource resource = validateAndFindHAResource(haConfig);
if (resource == null) {
continue;
}
final HAProvider<HAResource> haProvider = validateAndFindHAProvider(haConfig, resource);
if (haProvider == null) {
continue;
}
final HAResourceCounter counter = getHACounter(haConfig.getResourceId(), haConfig.getResourceType());
if (haConfig.getState() == HAConfig.HAState.Recovering) {
if (counter.canAttemptRecovery()) {
if (counter.getRecoveryCounter() >= (Long)(haProvider.getConfigValue(HAProviderConfig.MaxRecoveryAttempts, resource))) {
transitionHAState(HAConfig.Event.RecoveryOperationThresholdExceeded, haConfig);
continue;
}
final RecoveryTask task = ComponentContext.inject(new RecoveryTask(resource, haProvider, haConfig,
HAProviderConfig.RecoveryTimeout, recoveryExecutor));
final Future<Boolean> recoveryFuture = recoveryExecutor.submit(task);
counter.setRecoveryFuture(recoveryFuture);
counter.incrRecoveryCounter();
}
}
if (haConfig.getState() == HAConfig.HAState.Recovered) {
counter.markRecoveryStarted();
if (counter.canExitRecovery((Long)(haProvider.getConfigValue(HAProviderConfig.RecoveryWaitTimeout, resource)))) {
transitionHAState(HAConfig.Event.RecoveryWaitPeriodTimeout, haConfig);
counter.markRecoveryCompleted();
}
}
}
} catch (Throwable t) {
LOG.error("Error trying to perform recovery operation in HA manager", t);
}
}
}
private final class FencingPollTask extends ManagedContextRunnable implements BackgroundPollTask {
@Override
protected void runInContext() {
try {
if (LOG.isTraceEnabled()) {
LOG.trace("HA fencing task is running...");
}
final List<HAConfig> haConfigList = new ArrayList<HAConfig>(haConfigDao.listAll());
for (final HAConfig haConfig : haConfigList) {
if (!checkHAOwnership(haConfig)) {
continue;
}
final HAResource resource = validateAndFindHAResource(haConfig);
if (resource == null) {
continue;
}
final HAProvider<HAResource> haProvider = validateAndFindHAProvider(haConfig, resource);
if (haProvider == null) {
continue;
}
final HAResourceCounter counter = getHACounter(haConfig.getResourceId(), haConfig.getResourceType());
if (counter.lastFencingCompleted()) {
if (haConfig.getState() == HAConfig.HAState.Fencing) {
final FenceTask task = ComponentContext.inject(new FenceTask(resource, haProvider, haConfig,
HAProviderConfig.FenceTimeout, fenceExecutor));
final Future<Boolean> fenceFuture = fenceExecutor.submit(task);
counter.setFenceFuture(fenceFuture);
}
}
}
} catch (Throwable t) {
LOG.error("Error trying to perform fencing operation in HA manager", t);
}
}
}
}

128
server/src/org/apache/cloudstack/ha/HAResourceCounter.java Normal file
View File

@ -0,0 +1,128 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.ha;
import java.util.concurrent.Future;
import java.util.concurrent.atomic.AtomicLong;
public final class HAResourceCounter {
private AtomicLong activityCheckCounter = new AtomicLong(0);
private AtomicLong activityCheckFailureCounter = new AtomicLong(0);
private AtomicLong recoveryOperationCounter = new AtomicLong(0);
private Long firstHealthCheckFailureTimestamp;
private Long lastActivityCheckTimestamp;
private Long degradedTimestamp;
private Long recoverTimestamp;
private Future<Boolean> recoveryFuture;
private Future<Boolean> fenceFuture;
public long getActivityCheckCounter() {
return activityCheckCounter.get();
}
public long getRecoveryCounter() {
return recoveryOperationCounter.get();
}
public synchronized void incrActivityCounter(final boolean isFailure) {
lastActivityCheckTimestamp = System.currentTimeMillis();
activityCheckCounter.incrementAndGet();
if (isFailure) {
activityCheckFailureCounter.incrementAndGet();
}
}
public synchronized void incrRecoveryCounter() {
recoveryOperationCounter.incrementAndGet();
}
public synchronized void resetActivityCounter() {
activityCheckCounter.set(0);
activityCheckFailureCounter.set(0);
}
public synchronized void resetRecoveryCounter() {
recoverTimestamp = null;
recoveryFuture = null;
recoveryOperationCounter.set(0);
}
public synchronized void resetSuspectTimestamp() {
firstHealthCheckFailureTimestamp = null;
}
public boolean hasActivityThresholdExceeded(final double failureRatio) {
return activityCheckFailureCounter.get() > (activityCheckCounter.get() * failureRatio);
}
public boolean canPerformActivityCheck(final Long activityCheckInterval) {
return lastActivityCheckTimestamp == null || (System.currentTimeMillis() - lastActivityCheckTimestamp) > (activityCheckInterval * 1000);
}
public boolean canRecheckActivity(final Long maxDegradedPeriod) {
return degradedTimestamp == null || (System.currentTimeMillis() - degradedTimestamp) > (maxDegradedPeriod * 1000);
}
public boolean canExitRecovery(final Long maxRecoveryWaitPeriod) {
return recoverTimestamp != null && (System.currentTimeMillis() - recoverTimestamp) > (maxRecoveryWaitPeriod * 1000);
}
public long getSuspectTimeStamp() {
if (firstHealthCheckFailureTimestamp == null) {
firstHealthCheckFailureTimestamp = System.currentTimeMillis();
}
return firstHealthCheckFailureTimestamp;
}
public synchronized void markResourceSuspected() {
firstHealthCheckFailureTimestamp = System.currentTimeMillis();
}
public synchronized void markResourceDegraded() {
degradedTimestamp = System.currentTimeMillis();
}
public synchronized void markRecoveryStarted() {
if (recoverTimestamp == null) {
recoverTimestamp = System.currentTimeMillis();
}
}
public synchronized void markRecoveryCompleted() {
recoverTimestamp = null;
recoveryFuture = null;
}
public void setRecoveryFuture(final Future<Boolean> future) {
recoveryFuture = future;
}
public boolean canAttemptRecovery() {
return recoveryFuture == null || recoveryFuture.isDone();
}
public void setFenceFuture(final Future<Boolean> future) {
fenceFuture = future;
}
public boolean lastFencingCompleted() {
return fenceFuture == null || fenceFuture.isDone();
}
}

35
server/src/org/apache/cloudstack/ha/provider/ActivityCheckerInterface.java Normal file
View File

@ -0,0 +1,35 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.ha.provider;
import org.joda.time.DateTime;
import org.apache.cloudstack.ha.HAResource;
import com.cloud.utils.component.Adapter;
/**
* Checking activity requires deeper investigation. This will be invoked when a health check has failed.
*
* @param <R>
*/
public interface ActivityCheckerInterface<R extends HAResource> extends Adapter {
boolean isActive(R r, DateTime t) throws HACheckerException ;
}

29
server/src/org/apache/cloudstack/ha/provider/HACheckerException.java Normal file
View File

@ -0,0 +1,29 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.ha.provider;
public class HACheckerException extends Exception {
private static final long serialVersionUID = 1L;
public HACheckerException(String string, Exception e) {
super(string, e);
}
}

29
server/src/org/apache/cloudstack/ha/provider/HAFenceException.java Normal file
View File

@ -0,0 +1,29 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.ha.provider;
public class HAFenceException extends Exception {
private static final long serialVersionUID = 1L;
public HAFenceException(String string, Exception e) {
super(string, e);
}
}

65
server/src/org/apache/cloudstack/ha/provider/HAProvider.java Normal file
View File

@ -0,0 +1,65 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.ha.provider;
import com.cloud.utils.component.Adapter;
import org.apache.cloudstack.ha.HAConfig;
import org.joda.time.DateTime;
import org.apache.cloudstack.ha.HAResource;
public interface HAProvider<R extends HAResource> extends Adapter {
enum HAProviderConfig {
HealthCheckTimeout,
ActivityCheckTimeout,
RecoveryTimeout,
FenceTimeout,
ActivityCheckFailureRatio,
MaxActivityChecks,
MaxRecoveryAttempts,
MaxActivityCheckInterval,
MaxDegradedWaitTimeout,
RecoveryWaitTimeout
};
HAResource.ResourceType resourceType();
HAResource.ResourceSubType resourceSubType();
boolean isDisabled(R r);
boolean isInMaintenanceMode(R r);
boolean isEligible(R r);
boolean isHealthy(R r) throws HACheckerException;
boolean hasActivity(R r, DateTime afterThis) throws HACheckerException;
boolean recover(R r) throws HARecoveryException;
boolean fence(R r) throws HAFenceException;
void setFenced(R r);
void sendAlert(R r, HAConfig.HAState nextState);
Object getConfigValue(HAProviderConfig name, R r);
}

28
server/src/org/apache/cloudstack/ha/provider/HARecoveryException.java Normal file
View File

@ -0,0 +1,28 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.ha.provider;
public class HARecoveryException extends Exception {
private static final long serialVersionUID = 1L;
public HARecoveryException(String string, Exception e) {
super(string, e);
}
}

34
server/src/org/apache/cloudstack/ha/provider/HealthCheckerInterface.java Normal file
View File

@ -0,0 +1,34 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.ha.provider;
import org.apache.cloudstack.ha.HAResource;
import com.cloud.utils.component.Adapter;
/**
* Health checker is a quick way to find out if a resource is active. Like pinging the host or checking agent health.
*
* @param <R>
*/
public interface HealthCheckerInterface<R extends HAResource> extends Adapter {
boolean isHealthy(R r);
}

23
server/src/org/apache/cloudstack/ha/provider/HostHAProvider.java Normal file
View File

@ -0,0 +1,23 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.ha.provider;
import com.cloud.host.Host;
public interface HostHAProvider extends HAProvider<Host> {
}

105
server/src/org/apache/cloudstack/ha/provider/host/HAAbstractHostProvider.java Normal file
View File

@ -0,0 +1,105 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.ha.provider.host;
import com.cloud.agent.AgentManager;
import com.cloud.alert.AlertManager;
import com.cloud.ha.HighAvailabilityManager;
import com.cloud.host.Host;
import com.cloud.host.HostVO;
import com.cloud.host.Status;
import com.cloud.host.Status.Event;
import com.cloud.resource.ResourceManager;
import com.cloud.resource.ResourceState;
import com.cloud.utils.component.AdapterBase;
import com.cloud.utils.fsm.NoTransitionException;
import org.apache.cloudstack.alert.AlertService;
import org.apache.cloudstack.ha.HAConfig;
import org.apache.cloudstack.ha.HAResource;
import org.apache.cloudstack.ha.provider.HAProvider;
import org.apache.cloudstack.utils.identity.ManagementServerNode;
import org.apache.log4j.Logger;
import javax.inject.Inject;
public abstract class HAAbstractHostProvider extends AdapterBase implements HAProvider<Host> {
private final static Logger LOG = Logger.getLogger(HAAbstractHostProvider.class);
@Inject
private AlertManager alertManager;
@Inject
protected AgentManager agentManager;
@Inject
protected ResourceManager resourceManager;
@Inject
protected HighAvailabilityManager oldHighAvailabilityManager;
@Override
public HAResource.ResourceType resourceType() {
return HAResource.ResourceType.Host;
}
public HAResource.ResourceSubType resourceSubType() {
return HAResource.ResourceSubType.Unknown;
}
@Override
public boolean isDisabled(final Host host) {
return host.isDisabled();
}
@Override
public boolean isInMaintenanceMode(final Host host) {
return host.isInMaintenanceStates();
}
@Override
public void setFenced(final Host r) {
if (r.getState() != Status.Down) {
try {
LOG.debug("Trying to disconnect the host without investigation and scheduling HA for the VMs on host id=" + r.getId());
agentManager.disconnectWithoutInvestigation(r.getId(), Event.HostDown);
oldHighAvailabilityManager.scheduleRestartForVmsOnHost((HostVO)r, true);
} catch (Exception e) {
LOG.error("Failed to disconnect host and schedule HA restart of VMs after fencing the host: ", e);
}
try {
resourceManager.resourceStateTransitTo(r, ResourceState.Event.InternalEnterMaintenance, ManagementServerNode.getManagementServerId());
} catch (NoTransitionException e) {
LOG.error("Failed to put host in maintenance mode after host-ha fencing and scheduling VM-HA: ", e);
}
}
}
@Override
public void sendAlert(final Host host, final HAConfig.HAState nextState) {
String subject = "HA operation performed for host";
String body = subject;
if (HAConfig.HAState.Fencing.equals(nextState)) {
subject = String.format("HA Fencing of host id=%d, in dc id=%d performed", host.getId(), host.getDataCenterId());
body = String.format("HA Fencing has been performed for host id=%d, uuid=%s in datacenter id=%d", host.getId(), host.getUuid(), host.getDataCenterId());
} else if (HAConfig.HAState.Recovering.equals(nextState)) {
subject = String.format("HA Recovery of host id=%d, in dc id=%d performed", host.getId(), host.getDataCenterId());
body = String.format("HA Recovery has been performed for host id=%d, uuid=%s in datacenter id=%d", host.getId(), host.getUuid(), host.getDataCenterId());
}
alertManager.sendAlert(AlertService.AlertType.ALERT_TYPE_HA_ACTION, host.getDataCenterId(), host.getPodId(), subject, body);
}
}

82
server/src/org/apache/cloudstack/ha/task/ActivityCheckTask.java Normal file
View File

@ -0,0 +1,82 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.ha.task;
import org.apache.cloudstack.ha.HAConfig;
import org.apache.cloudstack.ha.HAManager;
import org.apache.cloudstack.ha.HAResource;
import org.apache.cloudstack.ha.HAResourceCounter;
import org.apache.cloudstack.ha.provider.HACheckerException;
import org.apache.cloudstack.ha.provider.HAProvider;
import org.apache.cloudstack.ha.provider.HAProvider.HAProviderConfig;
import org.apache.log4j.Logger;
import javax.inject.Inject;
import org.joda.time.DateTime;
import java.util.concurrent.ExecutorService;
public class ActivityCheckTask extends BaseHATask {
public static final Logger LOG = Logger.getLogger(ActivityCheckTask.class);
@Inject
private HAManager haManager;
private final long disconnectTime;
public ActivityCheckTask(final HAResource resource, final HAProvider<HAResource> haProvider, final HAConfig haConfig, final HAProvider.HAProviderConfig haProviderConfig,
final ExecutorService executor, final long disconnectTime) {
super(resource, haProvider, haConfig, haProviderConfig, executor);
this.disconnectTime = disconnectTime;
}
public boolean performAction() throws HACheckerException {
return getHaProvider().hasActivity(getResource(), new DateTime(disconnectTime));
}
public void processResult(boolean result, Throwable t) {
final HAConfig haConfig = getHaConfig();
final HAProvider<HAResource> haProvider = getHaProvider();
final HAResource resource = getResource();
final HAResourceCounter counter = haManager.getHACounter(haConfig.getResourceId(), haConfig.getResourceType());
if (t != null && t instanceof HACheckerException) {
haManager.transitionHAState(HAConfig.Event.Ineligible, getHaConfig());
counter.resetActivityCounter();
return;
}
counter.incrActivityCounter(!result);
long maxActivityChecks = (Long)haProvider.getConfigValue(HAProviderConfig.MaxActivityChecks, resource);
if (counter.getActivityCheckCounter() < maxActivityChecks) {
haManager.transitionHAState(HAConfig.Event.TooFewActivityCheckSamples, haConfig);
return;
}
double activityCheckFailureRatio = (Double)haProvider.getConfigValue(HAProviderConfig.ActivityCheckFailureRatio, resource);
if (counter.hasActivityThresholdExceeded(activityCheckFailureRatio)) {
haManager.transitionHAState(HAConfig.Event.ActivityCheckFailureOverThresholdRatio, haConfig);
} else {
haManager.transitionHAState(HAConfig.Event.ActivityCheckFailureUnderThresholdRatio, haConfig);
counter.markResourceDegraded();
}
counter.resetActivityCounter();
}
}

102
server/src/org/apache/cloudstack/ha/task/BaseHATask.java Normal file
View File

@ -0,0 +1,102 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.ha.task;
import org.apache.cloudstack.ha.HAConfig;
import org.apache.cloudstack.ha.HAResource;
import org.apache.cloudstack.ha.provider.HACheckerException;
import org.apache.cloudstack.ha.provider.HAFenceException;
import org.apache.cloudstack.ha.provider.HAProvider;
import org.apache.cloudstack.ha.provider.HARecoveryException;
import org.apache.log4j.Logger;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
public abstract class BaseHATask implements Callable<Boolean> {
public static final Logger LOG = Logger.getLogger(BaseHATask.class);
private final HAResource resource;
private final HAProvider<HAResource> haProvider;
private final HAConfig haConfig;
private final ExecutorService executor;
private Long timeout;
public BaseHATask(final HAResource resource, final HAProvider<HAResource> haProvider, final HAConfig haConfig, final HAProvider.HAProviderConfig haProviderConfig,
final ExecutorService executor) {
this.resource = resource;
this.haProvider = haProvider;
this.haConfig = haConfig;
this.executor = executor;
this.timeout = (Long)haProvider.getConfigValue(haProviderConfig, resource);
}
public HAProvider<HAResource> getHaProvider() {
return haProvider;
}
public HAConfig getHaConfig() {
return haConfig;
}
public HAResource getResource() {
return resource;
}
public String getTaskType() {
return this.getClass().getSimpleName();
}
public boolean performAction() throws HACheckerException, HAFenceException, HARecoveryException {
return true;
}
public abstract void processResult(boolean result, Throwable e);
@Override
public Boolean call() {
final Future<Boolean> future = executor.submit(new Callable<Boolean>() {
@Override
public Boolean call() throws HACheckerException, HAFenceException, HARecoveryException {
return performAction();
}
});
boolean result = false;
Throwable throwable = null;
try {
if (timeout == null) {
result = future.get();
} else {
result = future.get(timeout, TimeUnit.SECONDS);
}
} catch (InterruptedException | ExecutionException e) {
LOG.warn("Exception occurred while running " + getTaskType() + " on a resource: " + e.getMessage(), e.getCause());
throwable = e.getCause();
} catch (TimeoutException e) {
LOG.trace(getTaskType() + " operation timed out for resource id:" + resource.getId());
}
processResult(result, throwable);
return result;
}
}

55
server/src/org/apache/cloudstack/ha/task/FenceTask.java Normal file
View File

@ -0,0 +1,55 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.ha.task;
import org.apache.cloudstack.ha.HAConfig;
import org.apache.cloudstack.ha.HAManager;
import org.apache.cloudstack.ha.HAResource;
import org.apache.cloudstack.ha.HAResourceCounter;
import org.apache.cloudstack.ha.provider.HACheckerException;
import org.apache.cloudstack.ha.provider.HAFenceException;
import org.apache.cloudstack.ha.provider.HAProvider;
import javax.inject.Inject;
import java.util.concurrent.ExecutorService;
public class FenceTask extends BaseHATask {
@Inject
private HAManager haManager;
public FenceTask(final HAResource resource, final HAProvider<HAResource> haProvider, final HAConfig haConfig,
final HAProvider.HAProviderConfig haProviderConfig, final ExecutorService executor) {
super(resource, haProvider, haConfig, haProviderConfig, executor);
}
public boolean performAction() throws HACheckerException, HAFenceException {
return getHaProvider().fence(getResource());
}
public void processResult(boolean result, Throwable e) {
final HAConfig haConfig = getHaConfig();
final HAResourceCounter counter = haManager.getHACounter(haConfig.getResourceId(), haConfig.getResourceType());
if (result) {
counter.resetRecoveryCounter();
haManager.transitionHAState(HAConfig.Event.Fenced, haConfig);
getHaProvider().setFenced(getResource());
}
getHaProvider().sendAlert(getResource(), HAConfig.HAState.Fencing);
}
}

63
server/src/org/apache/cloudstack/ha/task/HealthCheckTask.java Normal file
View File

@ -0,0 +1,63 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.ha.task;
import org.apache.cloudstack.ha.HAConfig;
import org.apache.cloudstack.ha.HAManager;
import org.apache.cloudstack.ha.HAResource;
import org.apache.cloudstack.ha.HAResourceCounter;
import org.apache.cloudstack.ha.provider.HACheckerException;
import org.apache.cloudstack.ha.provider.HAProvider;
import org.apache.log4j.Logger;
import javax.inject.Inject;
import java.util.concurrent.ExecutorService;
public class HealthCheckTask extends BaseHATask {
@Inject
private HAManager haManager;
public static final Logger LOG = Logger.getLogger(HealthCheckTask.class);
public HealthCheckTask(final HAResource resource, final HAProvider<HAResource> haProvider, final HAConfig haConfig,
final HAProvider.HAProviderConfig haProviderConfig, final ExecutorService executor) {
super(resource, haProvider, haConfig, haProviderConfig, executor);
}
public boolean performAction() throws HACheckerException {
return getHaProvider().isHealthy(getResource());
}
public void processResult(boolean result, Throwable e) {
final HAConfig haConfig = getHaConfig();
final HAResourceCounter counter = haManager.getHACounter(haConfig.getResourceId(), haConfig.getResourceType());
if (result) {
haManager.transitionHAState(HAConfig.Event.HealthCheckPassed, haConfig);
if (haConfig.getState() == HAConfig.HAState.Fenced) {
haManager.disableHA(haConfig.getResourceId(), haConfig.getResourceType());
}
counter.resetSuspectTimestamp();
counter.resetActivityCounter();
counter.resetRecoveryCounter();
} else {
haManager.transitionHAState(HAConfig.Event.HealthCheckFailed, haConfig);
counter.markResourceSuspected();
}
}
}

51
server/src/org/apache/cloudstack/ha/task/RecoveryTask.java Normal file
View File

@ -0,0 +1,51 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.ha.task;
import org.apache.cloudstack.ha.HAConfig;
import org.apache.cloudstack.ha.HAManager;
import org.apache.cloudstack.ha.HAResource;
import org.apache.cloudstack.ha.provider.HACheckerException;
import org.apache.cloudstack.ha.provider.HAProvider;
import org.apache.cloudstack.ha.provider.HARecoveryException;
import javax.inject.Inject;
import java.util.concurrent.ExecutorService;
public class RecoveryTask extends BaseHATask {
@Inject
private HAManager haManager;
public RecoveryTask(final HAResource resource, final HAProvider<HAResource> haProvider, final HAConfig haConfig,
final HAProvider.HAProviderConfig haProviderConfig, final ExecutorService executor) {
super(resource, haProvider, haConfig, haProviderConfig, executor);
}
public boolean performAction() throws HACheckerException, HARecoveryException {
return getHaProvider().recover(getResource());
}
public void processResult(boolean result, Throwable e) {
final HAConfig haConfig = getHaConfig();
if (result) {
haManager.transitionHAState(HAConfig.Event.Recovered, haConfig);
}
getHaProvider().sendAlert(getResource(), HAConfig.HAState.Recovering);
}
}

156
server/src/org/apache/cloudstack/outofbandmanagement/OutOfBandManagementServiceImpl.java
View File

@ -31,7 +31,6 @@ import com.cloud.host.dao.HostDao;
import com.cloud.org.Cluster;
import com.cloud.utils.component.Manager;
import com.cloud.utils.component.ManagerBase;
import com.cloud.utils.db.GlobalLock;
import com.cloud.utils.db.Transaction;
import com.cloud.utils.db.TransactionCallback;
import com.cloud.utils.db.TransactionStatus;
@ -45,10 +44,13 @@ import org.apache.cloudstack.api.response.OutOfBandManagementResponse;
import org.apache.cloudstack.context.CallContext;
import org.apache.cloudstack.framework.config.ConfigKey;
import org.apache.cloudstack.framework.config.Configurable;
import org.apache.cloudstack.managed.context.ManagedContextRunnable;
import org.apache.cloudstack.outofbandmanagement.dao.OutOfBandManagementDao;
import org.apache.cloudstack.outofbandmanagement.driver.OutOfBandManagementDriverChangePasswordCommand;
import org.apache.cloudstack.outofbandmanagement.driver.OutOfBandManagementDriverPowerCommand;
import org.apache.cloudstack.outofbandmanagement.driver.OutOfBandManagementDriverResponse;
import org.apache.cloudstack.poll.BackgroundPollManager;
import org.apache.cloudstack.poll.BackgroundPollTask;
import org.apache.cloudstack.utils.identity.ManagementServerNode;
import org.apache.log4j.Logger;
import org.springframework.stereotype.Component;
@ -79,6 +81,8 @@ public class OutOfBandManagementServiceImpl extends ManagerBase implements OutOf
private HostDao hostDao;
@Inject
private AlertManager alertMgr;
@Inject
private BackgroundPollManager backgroundPollManager;
private String name;
private long serviceId;
@ -87,15 +91,10 @@ public class OutOfBandManagementServiceImpl extends ManagerBase implements OutOf
private final Map<String, OutOfBandManagementDriver> outOfBandManagementDriversMap = new HashMap<String, OutOfBandManagementDriver>();
private static final String OOBM_ENABLED_DETAIL = "outOfBandManagementEnabled";
private static final int ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_HOST = 120;
private static Cache<Long, Long> hostAlertCache;
private static ExecutorService backgroundSyncBlockingExecutor;
private String getOutOfBandManagementHostLock(long id) {
return "oobm.host." + id;
}
private void initializeDriversMap() {
if (outOfBandManagementDriversMap.isEmpty() && outOfBandManagementDrivers != null && outOfBandManagementDrivers.size() > 0) {
for (final OutOfBandManagementDriver driver : outOfBandManagementDrivers) {
@ -204,12 +203,14 @@ public class OutOfBandManagementServiceImpl extends ManagerBase implements OutOf
OutOfBandManagement.PowerState currentPowerState = outOfBandManagementHost.getPowerState();
try {
OutOfBandManagement.PowerState newPowerState = OutOfBandManagement.PowerState.getStateMachine().getNextState(currentPowerState, event);
boolean result = outOfBandManagementDao.updateState(currentPowerState, event, newPowerState, outOfBandManagementHost, null);
boolean result = OutOfBandManagement.PowerState.getStateMachine().transitTo(outOfBandManagementHost, event, null, outOfBandManagementDao);
if (result) {
final String message = String.format("Transitioned out-of-band management power state from:%s to:%s due to event:%s for the host id:%d", currentPowerState, newPowerState, event, outOfBandManagementHost.getHostId());
LOG.debug(message);
ActionEventUtils.onActionEvent(CallContext.current().getCallingUserId(), CallContext.current().getCallingAccountId(), Domain.ROOT_DOMAIN,
EventTypes.EVENT_HOST_OUTOFBAND_MANAGEMENT_POWERSTATE_TRANSITION, message);
if (newPowerState == OutOfBandManagement.PowerState.Unknown) {
ActionEventUtils.onActionEvent(CallContext.current().getCallingUserId(), CallContext.current().getCallingAccountId(), Domain.ROOT_DOMAIN,
EventTypes.EVENT_HOST_OUTOFBAND_MANAGEMENT_POWERSTATE_TRANSITION, message);
}
}
return result;
} catch (NoTransitionException ignored) {
@ -280,7 +281,7 @@ public class OutOfBandManagementServiceImpl extends ManagerBase implements OutOf
public void submitBackgroundPowerSyncTask(final Host host) {
if (host != null) {
backgroundSyncBlockingExecutor.submit(new OutOfBandManagementBackgroundTask(this, host, OutOfBandManagement.PowerOperation.STATUS));
backgroundSyncBlockingExecutor.submit(new PowerOperationTask(this, host, OutOfBandManagement.PowerOperation.STATUS));
}
}
@ -358,7 +359,7 @@ public class OutOfBandManagementServiceImpl extends ManagerBase implements OutOf
@Override
@ActionEvent(eventType = EventTypes.EVENT_HOST_OUTOFBAND_MANAGEMENT_CONFIGURE, eventDescription = "updating out-of-band management configuration")
public OutOfBandManagementResponse configureOutOfBandManagement(final Host host, final ImmutableMap<OutOfBandManagement.Option, String> options) {
public OutOfBandManagementResponse configure(final Host host, final ImmutableMap<OutOfBandManagement.Option, String> options) {
OutOfBandManagement outOfBandManagementConfig = outOfBandManagementDao.findByHost(host.getId());
if (outOfBandManagementConfig == null) {
outOfBandManagementConfig = outOfBandManagementDao.persist(new OutOfBandManagementVO(host.getId()));
@ -386,7 +387,7 @@ public class OutOfBandManagementServiceImpl extends ManagerBase implements OutOf
@Override
@ActionEvent(eventType = EventTypes.EVENT_HOST_OUTOFBAND_MANAGEMENT_ACTION, eventDescription = "issuing host out-of-band management action", async = true)
public OutOfBandManagementResponse executeOutOfBandManagementPowerOperation(final Host host, final OutOfBandManagement.PowerOperation powerOperation, final Long timeout) {
public OutOfBandManagementResponse executePowerOperation(final Host host, final OutOfBandManagement.PowerOperation powerOperation, final Long timeout) {
checkOutOfBandManagementEnabledByZoneClusterHost(host);
final OutOfBandManagement outOfBandManagementConfig = getConfigForHost(host);
final ImmutableMap<OutOfBandManagement.Option, String> options = getOptions(outOfBandManagementConfig);
@ -430,64 +431,51 @@ public class OutOfBandManagementServiceImpl extends ManagerBase implements OutOf
@Override
@ActionEvent(eventType = EventTypes.EVENT_HOST_OUTOFBAND_MANAGEMENT_CHANGE_PASSWORD, eventDescription = "updating out-of-band management password")
public OutOfBandManagementResponse changeOutOfBandManagementPassword(final Host host, final String newPassword) {
public OutOfBandManagementResponse changePassword(final Host host, final String newPassword) {
checkOutOfBandManagementEnabledByZoneClusterHost(host);
if (Strings.isNullOrEmpty(newPassword)) {
throw new CloudRuntimeException(String.format("Cannot change out-of-band management password as provided new-password is null or empty for the host %s.", host.getUuid()));
}
GlobalLock outOfBandManagementHostLock = GlobalLock.getInternLock(getOutOfBandManagementHostLock(host.getId()));
try {
if (outOfBandManagementHostLock.lock(ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_HOST)) {
try {
final OutOfBandManagement outOfBandManagementConfig = outOfBandManagementDao.findByHost(host.getId());
final ImmutableMap<OutOfBandManagement.Option, String> options = getOptions(outOfBandManagementConfig);
if (!(options.containsKey(OutOfBandManagement.Option.PASSWORD) && !Strings.isNullOrEmpty(options.get(OutOfBandManagement.Option.PASSWORD)))) {
throw new CloudRuntimeException(String.format("Cannot change out-of-band management password as we've no previously configured password for the host %s.", host.getUuid()));
}
final OutOfBandManagementDriver driver = getDriver(outOfBandManagementConfig);
final OutOfBandManagementDriverChangePasswordCommand cmd = new OutOfBandManagementDriverChangePasswordCommand(options, ActionTimeout.valueIn(host.getClusterId()), newPassword);
final OutOfBandManagementDriverResponse driverResponse;
try {
driverResponse = driver.execute(cmd);
} catch (Exception e) {
LOG.error("Out-of-band management change password failed due to driver error: " + e.getMessage());
throw new CloudRuntimeException(String.format("Failed to change out-of-band management password for host (%s) due to driver error: %s", host.getUuid(), e.getMessage()));
}
if (!driverResponse.isSuccess()) {
throw new CloudRuntimeException(String.format("Failed to change out-of-band management password for host (%s) with error: %s", host.getUuid(), driverResponse.getError()));
}
final boolean updatedConfigResult = Transaction.execute(new TransactionCallback<Boolean>() {
@Override
public Boolean doInTransaction(TransactionStatus status) {
OutOfBandManagement updatedOutOfBandManagementConfig = outOfBandManagementDao.findByHost(host.getId());
updatedOutOfBandManagementConfig.setPassword(newPassword);
return outOfBandManagementDao.update(updatedOutOfBandManagementConfig.getId(), (OutOfBandManagementVO) updatedOutOfBandManagementConfig);
}
});
if (!updatedConfigResult) {
LOG.error(String.format("Succeeded to change out-of-band management password but failed to updated in database the new password:%s for the host id:%d", newPassword, host.getId()));
}
final OutOfBandManagementResponse response = new OutOfBandManagementResponse();
response.setSuccess(updatedConfigResult && driverResponse.isSuccess());
response.setResultDescription(driverResponse.getResult());
response.setId(host.getUuid());
return response;
} finally {
outOfBandManagementHostLock.unlock();
}
} else {
LOG.error("Unable to acquire synchronization lock to change out-of-band management password for host id: " + host.getId());
throw new CloudRuntimeException(String.format("Unable to acquire lock to change out-of-band management password for host (%s), please try after some time.", host.getUuid()));
}
} finally {
outOfBandManagementHostLock.releaseRef();
final OutOfBandManagement outOfBandManagementConfig = outOfBandManagementDao.findByHost(host.getId());
final ImmutableMap<OutOfBandManagement.Option, String> options = getOptions(outOfBandManagementConfig);
if (!(options.containsKey(OutOfBandManagement.Option.PASSWORD) && !Strings.isNullOrEmpty(options.get(OutOfBandManagement.Option.PASSWORD)))) {
throw new CloudRuntimeException(String.format("Cannot change out-of-band management password as we've no previously configured password for the host %s.", host.getUuid()));
}
final OutOfBandManagementDriver driver = getDriver(outOfBandManagementConfig);
final OutOfBandManagementDriverChangePasswordCommand changePasswordCmd = new OutOfBandManagementDriverChangePasswordCommand(options, ActionTimeout.valueIn(host.getClusterId()), newPassword);
final boolean changePasswordResult = Transaction.execute(new TransactionCallback<Boolean>() {
@Override
public Boolean doInTransaction(TransactionStatus status) {
final OutOfBandManagement updatedOutOfBandManagementConfig = outOfBandManagementDao.findByHost(host.getId());
updatedOutOfBandManagementConfig.setPassword(newPassword);
boolean result = outOfBandManagementDao.update(updatedOutOfBandManagementConfig.getId(), (OutOfBandManagementVO) updatedOutOfBandManagementConfig);
if (!result) {
throw new CloudRuntimeException(String.format("Failed to change out-of-band management password for host (%s) in the database.", host.getUuid()));
}
final OutOfBandManagementDriverResponse driverResponse;
try {
driverResponse = driver.execute(changePasswordCmd);
} catch (Exception e) {
LOG.error("Out-of-band management change password failed due to driver error: " + e.getMessage());
throw new CloudRuntimeException(String.format("Failed to change out-of-band management password for host (%s) due to driver error: %s", host.getUuid(), e.getMessage()));
}
if (!driverResponse.isSuccess()) {
throw new CloudRuntimeException(String.format("Failed to change out-of-band management password for host (%s) with error: %s", host.getUuid(), driverResponse.getError()));
}
return result && driverResponse.isSuccess();
}
});
final OutOfBandManagementResponse response = new OutOfBandManagementResponse();
response.setSuccess(changePasswordResult );
response.setId(host.getUuid());
return response;
}
@Override
@ -518,7 +506,9 @@ public class OutOfBandManagementServiceImpl extends ManagerBase implements OutOf
0L, TimeUnit.MILLISECONDS,
new ArrayBlockingQueue<Runnable>(10 * poolSize, true), new ThreadPoolExecutor.CallerRunsPolicy());
LOG.info("Starting out-of-band management background sync executor with thread pool-size=" + poolSize + " and background sync thread interval=" + SyncThreadInterval.value() + "s");
backgroundPollManager.submitTask(new OutOfBandManagementPowerStatePollTask());
LOG.info("Starting out-of-band management background sync executor with thread pool-size=" + poolSize);
return true;
}
@ -531,7 +521,7 @@ public class OutOfBandManagementServiceImpl extends ManagerBase implements OutOf
@Override
public boolean stop() {
backgroundSyncBlockingExecutor.shutdown();
outOfBandManagementDao.expireOutOfBandManagementOwnershipByServer(getId());
outOfBandManagementDao.expireServerOwnership(getId());
return true;
}
@ -542,7 +532,7 @@ public class OutOfBandManagementServiceImpl extends ManagerBase implements OutOf
@Override
public ConfigKey<?>[] getConfigKeys() {
return new ConfigKey<?>[] {ActionTimeout, SyncThreadInterval, SyncThreadPoolSize};
return new ConfigKey<?>[] {ActionTimeout, SyncThreadPoolSize};
}
public List<OutOfBandManagementDriver> getOutOfBandManagementDrivers() {
@ -552,4 +542,36 @@ public class OutOfBandManagementServiceImpl extends ManagerBase implements OutOf
public void setOutOfBandManagementDrivers(List<OutOfBandManagementDriver> outOfBandManagementDrivers) {
this.outOfBandManagementDrivers = outOfBandManagementDrivers;
}
private final class OutOfBandManagementPowerStatePollTask extends ManagedContextRunnable implements BackgroundPollTask {
@Override
protected void runInContext() {
try {
if (LOG.isTraceEnabled()) {
LOG.trace("Host out-of-band management power state poll task is running...");
}
final List<OutOfBandManagementVO> outOfBandManagementHosts = outOfBandManagementDao.findAllByManagementServer(ManagementServerNode.getManagementServerId());
if (outOfBandManagementHosts == null || outOfBandManagementHosts.isEmpty()) {
return;
}
for (final OutOfBandManagement outOfBandManagementHost : outOfBandManagementHosts) {
final Host host = hostDao.findById(outOfBandManagementHost.getHostId());
if (host == null) {
continue;
}
if (isOutOfBandManagementEnabled(host)) {
submitBackgroundPowerSyncTask(host);
} else if (outOfBandManagementHost.getPowerState() != OutOfBandManagement.PowerState.Disabled) {
if (transitionPowerStateToDisabled(Collections.singletonList(host))) {
if (LOG.isDebugEnabled()) {
LOG.debug("Out-of-band management was disabled in zone/cluster/host, disabled power state for host id:" + host.getId());
}
}
}
}
} catch (Throwable t) {
LOG.error("Error trying to retrieve host out-of-band management stats", t);
}
}
}
}

Some files were not shown because too many files have changed in this diff Show More