diff --git a/server/src/com/cloud/consoleproxy/AgentBasedConsoleProxyManager.java b/server/src/com/cloud/consoleproxy/AgentBasedConsoleProxyManager.java index 3c6716ca64d..f94baa81e14 100755 --- a/server/src/com/cloud/consoleproxy/AgentBasedConsoleProxyManager.java +++ b/server/src/com/cloud/consoleproxy/AgentBasedConsoleProxyManager.java @@ -367,4 +367,12 @@ public class AgentBasedConsoleProxyManager implements ConsoleProxyManager, Virtu @Override public void prepareStop(VirtualMachineProfile profile) { } + + @Override + public boolean recreateNeeded( + VirtualMachineProfile profile, long hostId, + Commands cmds, ReservationContext context) { + // TODO Auto-generated method stub + return false; + } } diff --git a/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java b/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java index 61d6cd4c920..deb5595bbc7 100755 --- a/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java +++ b/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java @@ -2010,4 +2010,12 @@ public class ConsoleProxyManagerImpl implements ConsoleProxyManager, ConsoleProx @Override public void prepareStop(VirtualMachineProfile profile) { } + + @Override + public boolean recreateNeeded( + VirtualMachineProfile profile, long hostId, + Commands cmds, ReservationContext context) { + // TODO Auto-generated method stub + return false; + } } diff --git a/server/src/com/cloud/network/lb/ElasticLoadBalancerManagerImpl.java b/server/src/com/cloud/network/lb/ElasticLoadBalancerManagerImpl.java index 793698a6e1b..7285da80519 100644 --- a/server/src/com/cloud/network/lb/ElasticLoadBalancerManagerImpl.java +++ b/server/src/com/cloud/network/lb/ElasticLoadBalancerManagerImpl.java @@ -1026,4 +1026,13 @@ public class ElasticLoadBalancerManagerImpl implements public void prepareStop(VirtualMachineProfile profile) { } + + + @Override + public boolean recreateNeeded( + VirtualMachineProfile profile, long hostId, + Commands cmds, ReservationContext context) { + // TODO Auto-generated method stub + return false; + } } diff --git a/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java b/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java index f772cbd07c6..07d3395d0d2 100755 --- a/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java +++ b/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java @@ -3282,4 +3282,17 @@ public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplian } } } + + @Override + public boolean recreateNeeded( + VirtualMachineProfile profile, long hostId, + Commands cmds, ReservationContext context) { + //asssume that if failed to ssh into router, meaning router is crashed + CheckSshAnswer answer = (CheckSshAnswer) cmds.getAnswer("checkSsh"); + if (answer == null || !answer.getResult()) { + return true; + } + + return false; + } } diff --git a/server/src/com/cloud/storage/StorageManager.java b/server/src/com/cloud/storage/StorageManager.java index 386d8c7a8e9..b8f89fa766e 100755 --- a/server/src/com/cloud/storage/StorageManager.java +++ b/server/src/com/cloud/storage/StorageManager.java @@ -176,7 +176,7 @@ public interface StorageManager extends StorageService, Manager { void createCapacityEntry(StoragePoolVO storagePool, short capacityType, long allocated); - void prepare(VirtualMachineProfile vm, DeployDestination dest) throws StorageUnavailableException, InsufficientStorageCapacityException, ConcurrentOperationException; + void prepare(VirtualMachineProfile vm, DeployDestination dest, boolean recreate) throws StorageUnavailableException, InsufficientStorageCapacityException, ConcurrentOperationException; void release(VirtualMachineProfile profile); diff --git a/server/src/com/cloud/storage/StorageManagerImpl.java b/server/src/com/cloud/storage/StorageManagerImpl.java index 350a35d3ca1..26654ed54ca 100755 --- a/server/src/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/com/cloud/storage/StorageManagerImpl.java @@ -3170,7 +3170,7 @@ public class StorageManagerImpl implements StorageManager, Manager, ClusterManag } @Override - public void prepare(VirtualMachineProfile vm, DeployDestination dest) throws StorageUnavailableException, InsufficientStorageCapacityException { + public void prepare(VirtualMachineProfile vm, DeployDestination dest, boolean recreate) throws StorageUnavailableException, InsufficientStorageCapacityException { if (dest == null) { if (s_logger.isDebugEnabled()) { @@ -3190,7 +3190,11 @@ public class StorageManagerImpl implements StorageManager, Manager, ClusterManag if (dest.getStorageForDisks() != null) { assignedPool = dest.getStorageForDisks().get(vol); } - if (assignedPool != null) { + if (assignedPool == null && recreate) { + assignedPool = _storagePoolDao.findById(vol.getPoolId()); + + } + if (assignedPool != null || recreate) { Volume.State state = vol.getState(); if (state == Volume.State.Allocated || state == Volume.State.Creating) { recreateVols.add(vol); @@ -3228,6 +3232,12 @@ public class StorageManagerImpl implements StorageManager, Manager, ClusterManag for (VolumeVO vol : recreateVols) { VolumeVO newVol; + StoragePool existingPool = null; + if (recreate && (dest.getStorageForDisks() == null || dest.getStorageForDisks().get(vol) == null)) { + existingPool = _storagePoolDao.findById(vol.getPoolId()); + s_logger.debug("existing pool: " + existingPool.getId()); + } + if (vol.getState() == Volume.State.Allocated || vol.getState() == Volume.State.Creating) { newVol = vol; } else { @@ -3248,7 +3258,9 @@ public class StorageManagerImpl implements StorageManager, Manager, ClusterManag } catch (NoTransitionException e) { throw new CloudRuntimeException("Unable to create " + e.toString()); } - Pair created = createVolume(newVol, _diskOfferingDao.findById(newVol.getDiskOfferingId()), vm, vols, dest); + + Pair created = createVolume(newVol, _diskOfferingDao.findById(newVol.getDiskOfferingId()), vm, vols, dest, existingPool); + if (created == null) { Long poolId = newVol.getPoolId(); newVol.setPoolId(null); @@ -3304,7 +3316,7 @@ public class StorageManagerImpl implements StorageManager, Manager, ClusterManag } public Pair createVolume(VolumeVO toBeCreated, DiskOfferingVO offering, VirtualMachineProfile vm, List alreadyCreated, - DeployDestination dest) throws StorageUnavailableException { + DeployDestination dest, StoragePool sPool) throws StorageUnavailableException { if (s_logger.isDebugEnabled()) { s_logger.debug("Creating volume: " + toBeCreated); } @@ -3314,9 +3326,15 @@ public class StorageManagerImpl implements StorageManager, Manager, ClusterManag if (toBeCreated.getTemplateId() != null) { template = _templateDao.findById(toBeCreated.getTemplateId()); } + + StoragePool pool = null; + if (sPool != null) { + pool = sPool; + } else { + pool = dest.getStorageForDisks().get(toBeCreated); + } - if (dest.getStorageForDisks() != null) { - StoragePool pool = dest.getStorageForDisks().get(toBeCreated); + if (pool != null) { if (s_logger.isDebugEnabled()) { s_logger.debug("Trying to create in " + pool); } diff --git a/server/src/com/cloud/storage/secondary/SecondaryStorageManagerImpl.java b/server/src/com/cloud/storage/secondary/SecondaryStorageManagerImpl.java index e93be77d8ef..ec7f3e14e89 100755 --- a/server/src/com/cloud/storage/secondary/SecondaryStorageManagerImpl.java +++ b/server/src/com/cloud/storage/secondary/SecondaryStorageManagerImpl.java @@ -1467,4 +1467,13 @@ public class SecondaryStorageManagerImpl implements SecondaryStorageVmManager, V public void prepareStop(VirtualMachineProfile profile) { } + + @Override + public boolean recreateNeeded( + VirtualMachineProfile profile, long hostId, + Commands cmds, ReservationContext context) { + // TODO Auto-generated method stub + return false; + } + } diff --git a/server/src/com/cloud/vm/UserVmManagerImpl.java b/server/src/com/cloud/vm/UserVmManagerImpl.java index 324ce536bbb..fb1399b7abd 100755 --- a/server/src/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/com/cloud/vm/UserVmManagerImpl.java @@ -3685,5 +3685,11 @@ public class UserVmManagerImpl implements UserVmManager, UserVmService, Manager @Override public void prepareStop(VirtualMachineProfile profile) { } + @Override + public boolean recreateNeeded(VirtualMachineProfile profile, + long hostId, Commands cmds, ReservationContext context) { + // TODO Auto-generated method stub + return false; + } } diff --git a/server/src/com/cloud/vm/VirtualMachineGuru.java b/server/src/com/cloud/vm/VirtualMachineGuru.java index 82e42dd2af7..c3d8cc76f48 100644 --- a/server/src/com/cloud/vm/VirtualMachineGuru.java +++ b/server/src/com/cloud/vm/VirtualMachineGuru.java @@ -65,6 +65,8 @@ public interface VirtualMachineGuru { void finalizeExpunge(T vm); + boolean recreateNeeded(VirtualMachineProfile profile, long hostId, Commands cmds, ReservationContext context); + /** * Returns the id parsed from the name. If it cannot parse the name, * then return null. This method is used to determine if this is diff --git a/server/src/com/cloud/vm/VirtualMachineManagerImpl.java b/server/src/com/cloud/vm/VirtualMachineManagerImpl.java index 4bbb39ae216..4ea9f4e8672 100755 --- a/server/src/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/server/src/com/cloud/vm/VirtualMachineManagerImpl.java @@ -652,6 +652,7 @@ public class VirtualMachineManagerImpl implements VirtualMachineManager, Listene DataCenterDeployment originalPlan = plan; int retry = _retry; + boolean recreate = false; while (retry-- != 0) { // It's != so that it can match -1. if(reuseVolume){ @@ -746,7 +747,8 @@ public class VirtualMachineManagerImpl implements VirtualMachineManager, Listene } _networkMgr.prepare(vmProfile, dest, ctx); if (vm.getHypervisorType() != HypervisorType.BareMetal) { - _storageMgr.prepare(vmProfile, dest); + _storageMgr.prepare(vmProfile, dest, recreate); + recreate = false; } //since StorageMgr succeeded in volume creation, reuse Volume for further tries until current cluster has capacity if(!reuseVolume){ @@ -798,6 +800,7 @@ public class VirtualMachineManagerImpl implements VirtualMachineManager, Listene if (s_logger.isDebugEnabled()) { s_logger.info("The guru did not like the answers so stopping " + vm); } + StopCommand cmd = new StopCommand(vm.getInstanceName()); StopAnswer answer = (StopAnswer) _agentMgr.easySend(destHostId, cmd); if (answer == null || !answer.getResult()) { @@ -805,7 +808,11 @@ public class VirtualMachineManagerImpl implements VirtualMachineManager, Listene _haMgr.scheduleStop(vm, destHostId, WorkType.ForceStop); throw new ExecutionException("Unable to stop " + vm + " so we are unable to retry the start operation"); } - throw new ExecutionException("Unable to start " + vm + " due to error in finalizeStart, not retrying"); + if (vmGuru.recreateNeeded(vmProfile, destHostId, cmds, ctx)) { + recreate = true; + } else { + throw new ExecutionException("Unable to start " + vm + " due to error in finalizeStart, not retrying"); + } } } s_logger.info("Unable to start VM on " + dest.getHost() + " due to " + (startAnswer == null ? " no start answer" : startAnswer.getDetails())); diff --git a/server/test/com/cloud/vm/MockUserVmManagerImpl.java b/server/test/com/cloud/vm/MockUserVmManagerImpl.java index a66305eb61c..b97a8fe52da 100644 --- a/server/test/com/cloud/vm/MockUserVmManagerImpl.java +++ b/server/test/com/cloud/vm/MockUserVmManagerImpl.java @@ -289,6 +289,12 @@ public class MockUserVmManagerImpl implements UserVmManager, UserVmService, Mana } + @Override + public boolean recreateNeeded(VirtualMachineProfile profile, + long hostId, Commands cmds, ReservationContext context) { + // TODO Auto-generated method stub + return false; + } @Override public UserVm startVirtualMachine(DeployVMCmd cmd) throws InsufficientCapacityException, ConcurrentOperationException, ResourceUnavailableException {