From 44c1ef04a09f4ca19075240830152f3577541a13 Mon Sep 17 00:00:00 2001 From: Koushik Das Date: Tue, 4 Aug 2015 12:59:09 +0530 Subject: [PATCH] CLOUDSTACK-8704: Schedule restart of router VMs ahead of user VMs as part of HA VRs are scheduled for HA ahead of user VMs --- .../cloud/ha/HighAvailabilityManagerImpl.java | 49 +++++++-------- .../ha/HighAvailabilityManagerImplTest.java | 61 ++++++++++++++++--- 2 files changed, 74 insertions(+), 36 deletions(-) diff --git a/server/src/com/cloud/ha/HighAvailabilityManagerImpl.java b/server/src/com/cloud/ha/HighAvailabilityManagerImpl.java index 0a179a099e7..bec1fd58cc2 100644 --- a/server/src/com/cloud/ha/HighAvailabilityManagerImpl.java +++ b/server/src/com/cloud/ha/HighAvailabilityManagerImpl.java @@ -31,7 +31,6 @@ import javax.naming.ConfigurationException; import org.apache.log4j.Logger; import org.apache.log4j.NDC; - import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.managed.context.ManagedContext; @@ -231,7 +230,7 @@ public class HighAvailabilityManagerImpl extends ManagerBase implements HighAvai } if (host.getHypervisorType() == HypervisorType.VMware || host.getHypervisorType() == HypervisorType.Hyperv) { - s_logger.info("Don't restart for VMs on host " + host.getId() + " as the host is VMware host or on Hyperv Host"); + s_logger.info("Don't restart VMs on host " + host.getId() + " as it is a " + host.getHypervisorType().toString() + " host"); return; } @@ -242,16 +241,18 @@ public class HighAvailabilityManagerImpl extends ManagerBase implements HighAvai // send an email alert that the host is down StringBuilder sb = null; + List reorderedVMList = new ArrayList(); if ((vms != null) && !vms.isEmpty()) { sb = new StringBuilder(); - sb.append(" Starting HA on the following VMs: "); + sb.append(" Starting HA on the following VMs:"); // collect list of vm names for the alert email - VMInstanceVO vm = vms.get(0); - if (vm.isHaEnabled()) { - sb.append(" " + vm); - } - for (int i = 1; i < vms.size(); i++) { - vm = vms.get(i); + for (int i = 0; i < vms.size(); i++) { + VMInstanceVO vm = vms.get(i); + if (vm.getType() == VirtualMachine.Type.User) { + reorderedVMList.add(vm); + } else { + reorderedVMList.add(0, vm); + } if (vm.isHaEnabled()) { sb.append(" " + vm.getHostName()); } @@ -261,25 +262,21 @@ public class HighAvailabilityManagerImpl extends ManagerBase implements HighAvai // send an email alert that the host is down, include VMs HostPodVO podVO = _podDao.findById(host.getPodId()); String hostDesc = "name: " + host.getName() + " (id:" + host.getId() + "), availability zone: " + dcVO.getName() + ", pod: " + podVO.getName(); + _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_HOST, host.getDataCenterId(), host.getPodId(), "Host is down, " + hostDesc, + "Host [" + hostDesc + "] is down." + ((sb != null) ? sb.toString() : "")); - _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_HOST, host.getDataCenterId(), host.getPodId(), "Host is down, " + hostDesc, "Host [" + hostDesc + - "] is down." + - ((sb != null) ? sb.toString() : "")); - - if (vms != null) { - for (VMInstanceVO vm : vms) { - if (s_logger.isDebugEnabled()) { - s_logger.debug("Notifying HA Mgr of to restart vm " + vm.getId() + "-" + vm.getInstanceName()); - } - vm = _instanceDao.findByUuid(vm.getUuid()); - Long hostId = vm.getHostId(); - if (hostId != null && !hostId.equals(host.getId())) { - s_logger.debug("VM " + vm.getInstanceName() + " is not on down host " + host.getId() + " it is on other host " - + hostId + " VM HA is done"); - continue; - } - scheduleRestart(vm, investigate); + for (VMInstanceVO vm : reorderedVMList) { + if (s_logger.isDebugEnabled()) { + s_logger.debug("Notifying HA Mgr of to restart vm " + vm.getId() + "-" + vm.getInstanceName()); } + vm = _instanceDao.findByUuid(vm.getUuid()); + Long hostId = vm.getHostId(); + if (hostId != null && !hostId.equals(host.getId())) { + s_logger.debug("VM " + vm.getInstanceName() + " is not on down host " + host.getId() + " it is on other host " + + hostId + " VM HA is done"); + continue; + } + scheduleRestart(vm, investigate); } } diff --git a/server/test/com/cloud/ha/HighAvailabilityManagerImplTest.java b/server/test/com/cloud/ha/HighAvailabilityManagerImplTest.java index 75a43139d43..087c71a05ad 100644 --- a/server/test/com/cloud/ha/HighAvailabilityManagerImplTest.java +++ b/server/test/com/cloud/ha/HighAvailabilityManagerImplTest.java @@ -19,6 +19,7 @@ package com.cloud.ha; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; +import java.lang.reflect.Array; import java.lang.reflect.Field; import java.util.ArrayList; import java.util.Arrays; @@ -57,6 +58,7 @@ import com.cloud.storage.dao.GuestOSCategoryDao; import com.cloud.storage.dao.GuestOSDao; import com.cloud.user.AccountManager; import com.cloud.vm.VMInstanceVO; +import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachineManager; import com.cloud.vm.dao.VMInstanceDao; @@ -74,13 +76,10 @@ public class HighAvailabilityManagerImplTest { HostPodDao _podDao; @Mock ClusterDetailsDao _clusterDetailsDao; - @Mock ServiceOfferingDao _serviceOfferingDao; - @Mock ManagedContext _managedContext; - @Mock AgentManager _agentMgr; @Mock @@ -103,7 +102,6 @@ public class HighAvailabilityManagerImplTest { ConfigurationDao _configDao; @Mock VolumeOrchestrationService volumeMgr; - @Mock HostVO hostVO; @@ -113,12 +111,18 @@ public class HighAvailabilityManagerImplTest { public void setup() throws IllegalArgumentException, IllegalAccessException, NoSuchFieldException, SecurityException { highAvailabilityManager = new HighAvailabilityManagerImpl(); - for (Field injectField : HighAvailabilityManagerImpl.class - .getDeclaredFields()) { + for (Field injectField : HighAvailabilityManagerImpl.class.getDeclaredFields()) { if (injectField.isAnnotationPresent(Inject.class)) { injectField.setAccessible(true); - injectField.set(highAvailabilityManager, this.getClass() - .getDeclaredField(injectField.getName()).get(this)); + injectField.set(highAvailabilityManager, this.getClass().getDeclaredField(injectField.getName()).get(this)); + } else if (injectField.getName().equals("_workers")) { + injectField.setAccessible(true); + for (Class clz : HighAvailabilityManagerImpl.class.getDeclaredClasses()) { + if (clz.getName().equals("com.cloud.ha.HighAvailabilityManagerImpl$WorkerThread")) { + Object obj = Array.newInstance(clz, 0); + injectField.set(highAvailabilityManager, obj); + } + } } } } @@ -127,10 +131,47 @@ public class HighAvailabilityManagerImplTest { public void scheduleRestartForVmsOnHost() { Mockito.when(hostVO.getType()).thenReturn(Host.Type.Routing); Mockito.when(hostVO.getHypervisorType()).thenReturn(HypervisorType.KVM); - Mockito.when(_instanceDao.listByHostId(42l)).thenReturn( - Arrays.asList(Mockito.mock(VMInstanceVO.class))); + Mockito.when(_instanceDao.listByHostId(42l)).thenReturn(Arrays.asList(Mockito.mock(VMInstanceVO.class))); Mockito.when(_podDao.findById(Mockito.anyLong())).thenReturn(Mockito.mock(HostPodVO.class)); Mockito.when(_dcDao.findById(Mockito.anyLong())).thenReturn(Mockito.mock(DataCenterVO.class)); + + highAvailabilityManager.scheduleRestartForVmsOnHost(hostVO, true); + } + + @Test + public void scheduleRestartForVmsOnHostNotSupported() { + Mockito.when(hostVO.getType()).thenReturn(Host.Type.Routing); + Mockito.when(hostVO.getHypervisorType()).thenReturn(HypervisorType.VMware); + + highAvailabilityManager.scheduleRestartForVmsOnHost(hostVO, true); + } + + @Test + public void scheduleRestartForVmsOnHostNonEmptyVMList() { + Mockito.when(hostVO.getId()).thenReturn(1l); + Mockito.when(hostVO.getType()).thenReturn(Host.Type.Routing); + Mockito.when(hostVO.getHypervisorType()).thenReturn(HypervisorType.XenServer); + List vms = new ArrayList(); + VMInstanceVO vm1 = Mockito.mock(VMInstanceVO.class); + Mockito.when(vm1.getHostId()).thenReturn(1l); + Mockito.when(vm1.getInstanceName()).thenReturn("i-2-3-VM"); + Mockito.when(vm1.getType()).thenReturn(VirtualMachine.Type.User); + Mockito.when(vm1.isHaEnabled()).thenReturn(true); + vms.add(vm1); + VMInstanceVO vm2 = Mockito.mock(VMInstanceVO.class); + Mockito.when(vm2.getHostId()).thenReturn(1l); + Mockito.when(vm2.getInstanceName()).thenReturn("r-2-VM"); + Mockito.when(vm2.getType()).thenReturn(VirtualMachine.Type.DomainRouter); + Mockito.when(vm2.isHaEnabled()).thenReturn(true); + vms.add(vm2); + Mockito.when(_instanceDao.listByHostId(Mockito.anyLong())).thenReturn(vms); + Mockito.when(_instanceDao.findByUuid(vm1.getUuid())).thenReturn(vm1); + Mockito.when(_instanceDao.findByUuid(vm2.getUuid())).thenReturn(vm2); + Mockito.when(_podDao.findById(Mockito.anyLong())).thenReturn(Mockito.mock(HostPodVO.class)); + Mockito.when(_dcDao.findById(Mockito.anyLong())).thenReturn(Mockito.mock(DataCenterVO.class)); + Mockito.when(_haDao.findPreviousHA(Mockito.anyLong())).thenReturn(Arrays.asList(Mockito.mock(HaWorkVO.class))); + Mockito.when(_haDao.persist((HaWorkVO)Mockito.anyObject())).thenReturn(Mockito.mock(HaWorkVO.class)); + highAvailabilityManager.scheduleRestartForVmsOnHost(hostVO, true); }