etan39
/
cloudstack
mirror of https://github.com/apache/cloudstack.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2493 from shapeblue/fixmaintenance 

CLOUDSTACK-10326: Prevent hosts fall into Maintenance when there are running VMs on it
This commit is contained in:
Nicolas Vazquez 2018-06-20 12:00:58 -03:00 committed by GitHub
parent 39471c8c00 faf2a7760d
commit 539d7e10f3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 287 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
engine/schema/src/com/cloud/vm/dao/VMInstanceDao.java
View File

@ -150,4 +150,6 @@ public interface VMInstanceDao extends GenericDao<VMInstanceVO, Long>, StateDao<
VMInstanceVO findVMByHostNameInZone(String hostName, long zoneId);
boolean isPowerStateUpToDate(long instanceId);
List<VMInstanceVO> listNonMigratingVmsByHostEqualsLastHost(long hostId);
}

15
engine/schema/src/com/cloud/vm/dao/VMInstanceDaoImpl.java
View File

@ -93,6 +93,7 @@ public class VMInstanceDaoImpl extends GenericDaoBase<VMInstanceVO, Long> implem
protected GenericSearchBuilder<VMInstanceVO, String> DistinctHostNameSearch;
protected SearchBuilder<VMInstanceVO> HostAndStateSearch;
protected SearchBuilder<VMInstanceVO> StartingWithNoHostSearch;
protected SearchBuilder<VMInstanceVO> NotMigratingSearch;
@Inject
ResourceTagDao _tagsDao;
@ -280,6 +281,11 @@ public class VMInstanceDaoImpl extends GenericDaoBase<VMInstanceVO, Long> implem
DistinctHostNameSearch.join("nicSearch", nicSearch, DistinctHostNameSearch.entity().getId(), nicSearch.entity().getInstanceId(), JoinBuilder.JoinType.INNER);
DistinctHostNameSearch.done();
NotMigratingSearch = createSearchBuilder();
NotMigratingSearch.and("host", NotMigratingSearch.entity().getHostId(), Op.EQ);
NotMigratingSearch.and("lastHost", NotMigratingSearch.entity().getLastHostId(), Op.EQ);
NotMigratingSearch.and("state", NotMigratingSearch.entity().getState(), Op.NEQ);
NotMigratingSearch.done();
}
@Override
@ -304,6 +310,15 @@ public class VMInstanceDaoImpl extends GenericDaoBase<VMInstanceVO, Long> implem
return listBy(sc);
}
@Override
public List<VMInstanceVO> listNonMigratingVmsByHostEqualsLastHost(long hostId) {
SearchCriteria<VMInstanceVO> sc = NotMigratingSearch.create();
sc.setParameters("host", hostId);
sc.setParameters("lastHost", hostId);
sc.setParameters("state", State.Migrating);
return listBy(sc);
}
@Override
public List<VMInstanceVO> listByZoneId(long zoneId) {
SearchCriteria<VMInstanceVO> sc = AllFieldsSearch.create();

75
server/src/com/cloud/resource/ResourceManagerImpl.java
View File

@ -30,6 +30,7 @@ import java.util.Random;
import javax.inject.Inject;
import javax.naming.ConfigurationException;
import com.cloud.vm.dao.UserVmDetailsDao;
import org.apache.commons.lang.ObjectUtils;
import org.apache.log4j.Logger;
import org.springframework.stereotype.Component;
@ -54,6 +55,8 @@ import org.apache.commons.collections.CollectionUtils;
import com.cloud.agent.AgentManager;
import com.cloud.agent.api.Answer;
import com.cloud.agent.api.Command;
import com.cloud.agent.api.GetVncPortCommand;
import com.cloud.agent.api.GetVncPortAnswer;
import com.cloud.agent.api.GetGPUStatsAnswer;
import com.cloud.agent.api.GetGPUStatsCommand;
import com.cloud.agent.api.GetHostStatsAnswer;
@ -252,6 +255,8 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager,
private ConfigurationManager _configMgr;
@Inject
private ClusterVSMMapDao _clusterVSMMapDao;
@Inject
private UserVmDetailsDao userVmDetailsDao;
private final long _nodeId = ManagementServerNode.getManagementServerId();
@ -1287,6 +1292,68 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager,
}
}
/**
* Add VNC details as user VM details for each VM in 'vms' (KVM hosts only)
*/
protected void setKVMVncAccess(long hostId, List<VMInstanceVO> vms) {
for (VMInstanceVO vm : vms) {
GetVncPortAnswer vmVncPortAnswer = (GetVncPortAnswer) _agentMgr.easySend(hostId, new GetVncPortCommand(vm.getId(), vm.getInstanceName()));
if (vmVncPortAnswer != null) {
userVmDetailsDao.addDetail(vm.getId(), "kvm.vnc.address", vmVncPortAnswer.getAddress(), true);
userVmDetailsDao.addDetail(vm.getId(), "kvm.vnc.port", String.valueOf(vmVncPortAnswer.getPort()), true);
}
}
}
/**
* Configure VNC access for host VMs which have failed migrating to another host while trying to enter Maintenance mode
*/
protected void configureVncAccessForKVMHostFailedMigrations(HostVO host, List<VMInstanceVO> failedMigrations) {
if (host.getHypervisorType().equals(HypervisorType.KVM)) {
_agentMgr.pullAgentOutMaintenance(host.getId());
setKVMVncAccess(host.getId(), failedMigrations);
_agentMgr.pullAgentToMaintenance(host.getId());
}
}
/**
* Set host into ErrorInMaintenance state, as errors occurred during VM migrations. Do the following:
* - Cancel scheduled migrations for those which have already failed
* - Configure VNC access for VMs (KVM hosts only)
*/
protected boolean setHostIntoErrorInMaintenance(HostVO host, List<VMInstanceVO> failedMigrations) throws NoTransitionException {
s_logger.debug("Unable to migrate " + failedMigrations.size() + " VM(s) from host " + host.getUuid());
_haMgr.cancelScheduledMigrations(host);
configureVncAccessForKVMHostFailedMigrations(host, failedMigrations);
resourceStateTransitTo(host, ResourceState.Event.UnableToMigrate, _nodeId);
return false;
}
/**
* Safely transit host into Maintenance mode
*/
protected boolean setHostIntoMaintenance(HostVO host) throws NoTransitionException {
s_logger.debug("Host " + host.getUuid() + " entering in Maintenance");
resourceStateTransitTo(host, ResourceState.Event.InternalEnterMaintenance, _nodeId);
ActionEventUtils.onCompletedActionEvent(CallContext.current().getCallingUserId(), CallContext.current().getCallingAccountId(),
EventVO.LEVEL_INFO, EventTypes.EVENT_MAINTENANCE_PREPARE,
"completed maintenance for host " + host.getId(), 0);
return true;
}
/**
* Return true if host goes into Maintenance mode, only when:
* - No Running, Migrating or Failed migrations (host_id = last_host_id) for the host
*/
protected boolean isHostInMaintenance(HostVO host, List<VMInstanceVO> runningVms, List<VMInstanceVO> migratingVms, List<VMInstanceVO> failedMigrations) throws NoTransitionException {
if (CollectionUtils.isEmpty(runningVms) && CollectionUtils.isEmpty(migratingVms)) {
return CollectionUtils.isEmpty(failedMigrations) ?
setHostIntoMaintenance(host) :
setHostIntoErrorInMaintenance(host, failedMigrations);
}
return false;
}
@Override
public boolean checkAndMaintain(final long hostId) {
boolean hostInMaintenance = false;
@ -1296,11 +1363,9 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager,
if (host.getType() != Host.Type.Storage) {
final List<VMInstanceVO> vos = _vmDao.listByHostId(hostId);
final List<VMInstanceVO> vosMigrating = _vmDao.listVmsMigratingFromHost(hostId);
if (vos.isEmpty() && vosMigrating.isEmpty()) {
resourceStateTransitTo(host, ResourceState.Event.InternalEnterMaintenance, _nodeId);
hostInMaintenance = true;
ActionEventUtils.onCompletedActionEvent(CallContext.current().getCallingUserId(), CallContext.current().getCallingAccountId(), EventVO.LEVEL_INFO, EventTypes.EVENT_MAINTENANCE_PREPARE, "completed maintenance for host " + hostId, 0);
}
final List<VMInstanceVO> failedVmMigrations = _vmDao.listNonMigratingVmsByHostEqualsLastHost(hostId);
hostInMaintenance = isHostInMaintenance(host, vos, vosMigrating, failedVmMigrations);
}
} catch (final NoTransitionException e) {
s_logger.debug("Cannot transmit host " + host.getId() + "to Maintenance state", e);

10
server/src/com/cloud/servlet/ConsoleProxyServlet.java
View File

@ -35,6 +35,7 @@ import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.servlet.http.HttpSession;
import com.cloud.resource.ResourceState;
import org.apache.commons.codec.binary.Base64;
import org.apache.log4j.Logger;
import org.springframework.stereotype.Component;
@ -418,7 +419,14 @@ public class ConsoleProxyServlet extends HttpServlet {
StringBuffer sb = new StringBuffer(rootUrl);
String host = hostVo.getPrivateIpAddress();
Pair<String, Integer> portInfo = _ms.getVncPort(vm);
Pair<String, Integer> portInfo;
if (hostVo.getResourceState().equals(ResourceState.ErrorInMaintenance)) {
UserVmDetailVO detailAddress = _userVmDetailsDao.findDetail(vm.getId(), "kvm.vnc.address");
UserVmDetailVO detailPort = _userVmDetailsDao.findDetail(vm.getId(), "kvm.vnc.port");
portInfo = new Pair<>(detailAddress.getValue(), Integer.valueOf(detailPort.getValue()));
} else {
portInfo = _ms.getVncPort(vm);
}
if (s_logger.isDebugEnabled())
s_logger.debug("Port info " + portInfo.first());

191
server/test/com/cloud/resource/ResourceManagerImplTest.java Normal file
View File

@ -0,0 +1,191 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package com.cloud.resource;
import com.cloud.agent.AgentManager;
import com.cloud.agent.api.GetVncPortAnswer;
import com.cloud.agent.api.GetVncPortCommand;
import com.cloud.capacity.dao.CapacityDao;
import com.cloud.event.ActionEventUtils;
import com.cloud.ha.HighAvailabilityManager;
import com.cloud.host.Host;
import com.cloud.host.HostVO;
import com.cloud.host.dao.HostDao;
import com.cloud.hypervisor.Hypervisor;
import com.cloud.storage.StorageManager;
import com.cloud.utils.fsm.NoTransitionException;
import com.cloud.vm.VMInstanceVO;
import com.cloud.vm.dao.UserVmDetailsDao;
import com.cloud.vm.dao.VMInstanceDao;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.BDDMockito;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.MockitoAnnotations;
import org.mockito.Spy;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import static com.cloud.resource.ResourceState.Event.InternalEnterMaintenance;
import static com.cloud.resource.ResourceState.Event.UnableToMigrate;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyBoolean;
import static org.mockito.Matchers.anyLong;
import static org.mockito.Matchers.anyString;
import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@RunWith(PowerMockRunner.class)
@PrepareForTest({ActionEventUtils.class, ResourceManagerImpl.class})
public class ResourceManagerImplTest {
@Mock
private CapacityDao capacityDao;
@Mock
private StorageManager storageManager;
@Mock
private HighAvailabilityManager haManager;
@Mock
private UserVmDetailsDao userVmDetailsDao;
@Mock
private AgentManager agentManager;
@Mock
private HostDao hostDao;
@Mock
private VMInstanceDao vmInstanceDao;
@Spy
@InjectMocks
private ResourceManagerImpl resourceManager = new ResourceManagerImpl();
@Mock
private HostVO host;
@Mock
private VMInstanceVO vm1;
@Mock
private VMInstanceVO vm2;
@Mock
private GetVncPortAnswer getVncPortAnswerVm1;
@Mock
private GetVncPortAnswer getVncPortAnswerVm2;
@Mock
private GetVncPortCommand getVncPortCommandVm1;
@Mock
private GetVncPortCommand getVncPortCommandVm2;
private static long hostId = 1L;
private static long vm1Id = 1L;
private static String vm1InstanceName = "i-1-VM";
private static long vm2Id = 2L;
private static String vm2InstanceName = "i-2-VM";
private static String vm1VncAddress = "10.2.2.2";
private static int vm1VncPort = 5900;
private static String vm2VncAddress = "10.2.2.2";
private static int vm2VncPort = 5901;
@Before
public void setup() throws Exception {
MockitoAnnotations.initMocks(this);
when(host.getType()).thenReturn(Host.Type.Routing);
when(host.getId()).thenReturn(hostId);
when(host.getResourceState()).thenReturn(ResourceState.Enabled);
when(host.getHypervisorType()).thenReturn(Hypervisor.HypervisorType.VMware);
when(hostDao.findById(hostId)).thenReturn(host);
when(vm1.getId()).thenReturn(vm1Id);
when(vm2.getId()).thenReturn(vm2Id);
when(vm1.getInstanceName()).thenReturn(vm1InstanceName);
when(vm2.getInstanceName()).thenReturn(vm2InstanceName);
when(vmInstanceDao.listByHostId(hostId)).thenReturn(new ArrayList<>());
when(vmInstanceDao.listVmsMigratingFromHost(hostId)).thenReturn(new ArrayList<>());
when(vmInstanceDao.listNonMigratingVmsByHostEqualsLastHost(hostId)).thenReturn(new ArrayList<>());
PowerMockito.mockStatic(ActionEventUtils.class);
BDDMockito.given(ActionEventUtils.onCompletedActionEvent(anyLong(), anyLong(), anyString(), anyString(), anyString(), anyLong()))
.willReturn(1L);
when(getVncPortAnswerVm1.getAddress()).thenReturn(vm1VncAddress);
when(getVncPortAnswerVm1.getPort()).thenReturn(vm1VncPort);
when(getVncPortAnswerVm2.getAddress()).thenReturn(vm2VncAddress);
when(getVncPortAnswerVm2.getPort()).thenReturn(vm2VncPort);
PowerMockito.whenNew(GetVncPortCommand.class).withArguments(vm1Id, vm1InstanceName).thenReturn(getVncPortCommandVm1);
PowerMockito.whenNew(GetVncPortCommand.class).withArguments(vm2Id, vm2InstanceName).thenReturn(getVncPortCommandVm2);
when(agentManager.easySend(eq(hostId), eq(getVncPortCommandVm1))).thenReturn(getVncPortAnswerVm1);
when(agentManager.easySend(eq(hostId), eq(getVncPortCommandVm2))).thenReturn(getVncPortAnswerVm2);
}
@Test
public void testCheckAndMaintainEnterMaintenanceMode() throws NoTransitionException {
boolean enterMaintenanceMode = resourceManager.checkAndMaintain(hostId);
verify(resourceManager).isHostInMaintenance(host, new ArrayList<>(), new ArrayList<>(), new ArrayList<>());
verify(resourceManager).setHostIntoMaintenance(host);
verify(resourceManager).resourceStateTransitTo(eq(host), eq(InternalEnterMaintenance), anyLong());
Assert.assertTrue(enterMaintenanceMode);
}
@Test
public void testCheckAndMaintainErrorInMaintenanceRunningVms() throws NoTransitionException {
when(vmInstanceDao.listByHostId(hostId)).thenReturn(Arrays.asList(vm1, vm2));
boolean enterMaintenanceMode = resourceManager.checkAndMaintain(hostId);
verify(resourceManager).isHostInMaintenance(host, Arrays.asList(vm1, vm2), new ArrayList<>(), new ArrayList<>());
Assert.assertFalse(enterMaintenanceMode);
}
@Test
public void testCheckAndMaintainErrorInMaintenanceMigratingVms() throws NoTransitionException {
when(vmInstanceDao.listVmsMigratingFromHost(hostId)).thenReturn(Arrays.asList(vm1, vm2));
boolean enterMaintenanceMode = resourceManager.checkAndMaintain(hostId);
verify(resourceManager).isHostInMaintenance(host, new ArrayList<>(), Arrays.asList(vm1, vm2), new ArrayList<>());
Assert.assertFalse(enterMaintenanceMode);
}
@Test
public void testCheckAndMaintainErrorInMaintenanceFailedMigrations() throws NoTransitionException {
when(vmInstanceDao.listNonMigratingVmsByHostEqualsLastHost(hostId)).thenReturn(Arrays.asList(vm1, vm2));
boolean enterMaintenanceMode = resourceManager.checkAndMaintain(hostId);
verify(resourceManager).isHostInMaintenance(host, new ArrayList<>(), new ArrayList<>(), Arrays.asList(vm1, vm2));
verify(resourceManager).setHostIntoErrorInMaintenance(host, Arrays.asList(vm1, vm2));
verify(resourceManager).resourceStateTransitTo(eq(host), eq(UnableToMigrate), anyLong());
Assert.assertFalse(enterMaintenanceMode);
}
@Test
public void testConfigureVncAccessForKVMHostFailedMigrations() {
when(host.getHypervisorType()).thenReturn(Hypervisor.HypervisorType.KVM);
List<VMInstanceVO> vms = Arrays.asList(vm1, vm2);
resourceManager.configureVncAccessForKVMHostFailedMigrations(host, vms);
verify(agentManager).pullAgentOutMaintenance(hostId);
verify(resourceManager).setKVMVncAccess(hostId, vms);
verify(agentManager, times(vms.size())).easySend(eq(hostId), any(GetVncPortCommand.class));
verify(userVmDetailsDao).addDetail(eq(vm1Id), eq("kvm.vnc.address"), eq(vm1VncAddress), anyBoolean());
verify(userVmDetailsDao).addDetail(eq(vm1Id), eq("kvm.vnc.port"), eq(String.valueOf(vm1VncPort)), anyBoolean());
verify(userVmDetailsDao).addDetail(eq(vm2Id), eq("kvm.vnc.address"), eq(vm2VncAddress), anyBoolean());
verify(userVmDetailsDao).addDetail(eq(vm2Id), eq("kvm.vnc.port"), eq(String.valueOf(vm2VncPort)), anyBoolean());
verify(agentManager).pullAgentToMaintenance(hostId);
}
}