diff --git a/api/src/main/java/com/cloud/storage/Volume.java b/api/src/main/java/com/cloud/storage/Volume.java index 4a14197bd30..308ed2544ed 100644 --- a/api/src/main/java/com/cloud/storage/Volume.java +++ b/api/src/main/java/com/cloud/storage/Volume.java @@ -39,30 +39,38 @@ public interface Volume extends ControlledEntity, Identity, InternalIdentity, Ba }; enum State { - Allocated("The volume is allocated but has not been created yet."), - Creating("The volume is being created. getPoolId() should reflect the pool where it is being created."), - Ready("The volume is ready to be used."), - Migrating("The volume is migrating to other storage pool"), - Snapshotting("There is a snapshot created on this volume, not backed up to secondary storage yet"), - RevertSnapshotting("There is a snapshot created on this volume, the volume is being reverting from snapshot"), - Resizing("The volume is being resized"), - Expunging("The volume is being expunging"), - Expunged("The volume has been expunged, and can no longer be recovered"), - Destroy("The volume is destroyed, and can be recovered."), - Destroying("The volume is destroying, and can't be recovered."), - UploadOp("The volume upload operation is in progress or in short the volume is on secondary storage"), - Copying("Volume is copying from image store to primary, in case it's an uploaded volume"), - Uploaded("Volume is uploaded"), - NotUploaded("The volume entry is just created in DB, not yet uploaded"), - UploadInProgress("Volume upload is in progress"), - UploadError("Volume upload encountered some error"), - UploadAbandoned("Volume upload is abandoned since the upload was never initiated within a specified time"), - Attaching("The volume is attaching to a VM from Ready state."), - Restoring("The volume is being restored from backup."); + Allocated(false, "The volume is allocated but has not been created yet."), + Creating(true, "The volume is being created. getPoolId() should reflect the pool where it is being created."), + Ready(false, "The volume is ready to be used."), + Migrating(true, "The volume is migrating to other storage pool"), + Snapshotting(true, "There is a snapshot created on this volume, not backed up to secondary storage yet"), + RevertSnapshotting(true, "There is a snapshot created on this volume, the volume is being reverting from snapshot"), + Resizing(true, "The volume is being resized"), + Expunging(true, "The volume is being expunging"), + Expunged(false, "The volume has been expunged, and can no longer be recovered"), + Destroy(false, "The volume is destroyed, and can be recovered."), + Destroying(false, "The volume is destroying, and can't be recovered."), + UploadOp(true, "The volume upload operation is in progress or in short the volume is on secondary storage"), + Copying(true, "Volume is copying from image store to primary, in case it's an uploaded volume"), + Uploaded(false, "Volume is uploaded"), + NotUploaded(true, "The volume entry is just created in DB, not yet uploaded"), + UploadInProgress(true, "Volume upload is in progress"), + UploadError(false, "Volume upload encountered some error"), + UploadAbandoned(false, "Volume upload is abandoned since the upload was never initiated within a specified time"), + Attaching(true, "The volume is attaching to a VM from Ready state."), + Restoring(true, "The volume is being restored from backup."); + + boolean _transitional; String _description; - private State(String description) { + /** + * Volume State + * @param transitional true for transition/non-final state, otherwise false + * @param description description of the state + */ + private State(boolean transitional, String description) { + _transitional = transitional; _description = description; } @@ -70,6 +78,10 @@ public interface Volume extends ControlledEntity, Identity, InternalIdentity, Ba return s_fsm; } + public boolean isTransitional() { + return _transitional; + } + public String getDescription() { return _description; } diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java index c691b8b0942..2005b70b439 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java @@ -50,6 +50,7 @@ import com.cloud.network.rules.LoadBalancerContainer.Scheme; import com.cloud.offering.NetworkOffering; import com.cloud.user.Account; import com.cloud.user.User; +import com.cloud.utils.fsm.NoTransitionException; import com.cloud.utils.Pair; import com.cloud.vm.Nic; import com.cloud.vm.NicProfile; @@ -268,6 +269,8 @@ public interface NetworkOrchestrationService { Map finalizeServicesAndProvidersForNetwork(NetworkOffering offering, Long physicalNetworkId); + boolean stateTransitTo(Network network, Network.Event e) throws NoTransitionException; + List getProvidersForServiceInNetwork(Network network, Service service); StaticNatServiceProvider getStaticNatProviderForNetwork(Network network); diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VolumeService.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VolumeService.java index 2c12b70e9eb..50aee83f497 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VolumeService.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VolumeService.java @@ -18,13 +18,13 @@ */ package org.apache.cloudstack.engine.subsystem.api.storage; -import com.cloud.agent.api.Answer; import java.util.Map; import org.apache.cloudstack.engine.cloud.entity.api.VolumeEntity; import org.apache.cloudstack.framework.async.AsyncCallFuture; import org.apache.cloudstack.storage.command.CommandResult; +import com.cloud.agent.api.Answer; import com.cloud.agent.api.to.VirtualMachineTO; import com.cloud.exception.StorageAccessException; import com.cloud.host.Host; @@ -35,6 +35,9 @@ import com.cloud.user.Account; import com.cloud.utils.Pair; public interface VolumeService { + + String SNAPSHOT_ID = "SNAPSHOT_ID"; + class VolumeApiResult extends CommandResult { private final VolumeInfo volume; diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java index 0f6dfe60866..6c10a02abba 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java @@ -4424,7 +4424,8 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra return accessDetails; } - protected boolean stateTransitTo(final NetworkVO network, final Network.Event e) throws NoTransitionException { + @Override + public boolean stateTransitTo(final Network network, final Network.Event e) throws NoTransitionException { return _stateMachine.transitTo(network, e, null, _networksDao); } diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java index 5ebee87acd4..9e41e0d4d0e 100644 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java @@ -851,7 +851,7 @@ public class VolumeObject implements VolumeInfo { @Override public boolean delete() { - return dataStore == null ? true : dataStore.delete(this); + return dataStore == null || dataStore.delete(this); } @Override diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java index c0ef227251c..8a3cd39ecbf 100644 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java @@ -32,10 +32,6 @@ import java.util.concurrent.ExecutionException; import javax.inject.Inject; -import org.apache.cloudstack.secret.dao.PassphraseDao; -import com.cloud.storage.VMTemplateVO; -import com.cloud.storage.dao.VMTemplateDao; -import com.cloud.storage.resource.StorageProcessor; import org.apache.cloudstack.annotation.AnnotationService; import org.apache.cloudstack.annotation.dao.AnnotationDao; import org.apache.cloudstack.engine.cloud.entity.api.VolumeEntity; @@ -66,6 +62,7 @@ import org.apache.cloudstack.framework.async.AsyncCallbackDispatcher; import org.apache.cloudstack.framework.async.AsyncCompletionCallback; import org.apache.cloudstack.framework.async.AsyncRpcContext; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.secret.dao.PassphraseDao; import org.apache.cloudstack.storage.RemoteHostEndPoint; import org.apache.cloudstack.storage.command.CommandResult; import org.apache.cloudstack.storage.command.CopyCmdAnswer; @@ -83,6 +80,7 @@ import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreVO; import org.apache.cloudstack.storage.image.store.TemplateObject; import org.apache.cloudstack.storage.to.TemplateObjectTO; import org.apache.cloudstack.storage.to.VolumeObjectTO; +import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -122,13 +120,16 @@ import com.cloud.storage.StoragePool; import com.cloud.storage.VMTemplateStoragePoolVO; import com.cloud.storage.VMTemplateStorageResourceAssoc; import com.cloud.storage.VMTemplateStorageResourceAssoc.Status; +import com.cloud.storage.VMTemplateVO; import com.cloud.storage.Volume; import com.cloud.storage.Volume.State; import com.cloud.storage.VolumeDetailVO; import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VMTemplatePoolDao; import com.cloud.storage.dao.VolumeDao; import com.cloud.storage.dao.VolumeDetailsDao; +import com.cloud.storage.resource.StorageProcessor; import com.cloud.storage.snapshot.SnapshotApiService; import com.cloud.storage.snapshot.SnapshotManager; import com.cloud.storage.template.TemplateConstants; @@ -142,7 +143,6 @@ import com.cloud.utils.db.DB; import com.cloud.utils.db.GlobalLock; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.VirtualMachine; -import org.apache.commons.lang3.StringUtils; @Component public class VolumeServiceImpl implements VolumeService { @@ -206,8 +206,6 @@ public class VolumeServiceImpl implements VolumeService { @Inject private PassphraseDao passphraseDao; - private final static String SNAPSHOT_ID = "SNAPSHOT_ID"; - public VolumeServiceImpl() { } diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java index 9100ee5e34b..3c0f81d0bc1 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java @@ -38,9 +38,13 @@ import javax.naming.ConfigurationException; import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotService; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeService; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; import org.apache.cloudstack.framework.jobs.AsyncJob; @@ -65,7 +69,12 @@ import org.apache.log4j.MDC; import org.apache.log4j.NDC; import com.cloud.cluster.ClusterManagerListener; +import com.cloud.network.Network; +import com.cloud.network.dao.NetworkDao; +import com.cloud.network.dao.NetworkVO; import com.cloud.storage.Snapshot; +import com.cloud.storage.Volume; +import com.cloud.storage.VolumeDetailVO; import com.cloud.storage.dao.SnapshotDao; import com.cloud.storage.dao.SnapshotDetailsDao; import com.cloud.storage.dao.SnapshotDetailsVO; @@ -93,7 +102,11 @@ import com.cloud.utils.db.TransactionCallbackNoReturn; import com.cloud.utils.db.TransactionStatus; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.exception.ExceptionUtil; +import com.cloud.utils.fsm.NoTransitionException; import com.cloud.utils.mgmt.JmxUtil; +import com.cloud.vm.VMInstanceVO; +import com.cloud.vm.VirtualMachine; +import com.cloud.vm.VirtualMachineManager; import com.cloud.vm.dao.VMInstanceDao; public class AsyncJobManagerImpl extends ManagerBase implements AsyncJobManager, ClusterManagerListener, Configurable { @@ -148,6 +161,15 @@ public class AsyncJobManagerImpl extends ManagerBase implements AsyncJobManager, @Inject private SnapshotDetailsDao _snapshotDetailsDao; + @Inject + private VolumeDataFactory volFactory; + @Inject + private VirtualMachineManager virtualMachineManager; + @Inject + private NetworkDao networkDao; + @Inject + private NetworkOrchestrationService networkOrchestrationService; + private volatile long _executionRunNumber = 1; private final ScheduledExecutorService _heartbeatScheduler = Executors.newScheduledThreadPool(1, new NamedThreadFactory("AsyncJobMgr-Heartbeat")); @@ -1089,6 +1111,7 @@ public class AsyncJobManagerImpl extends ManagerBase implements AsyncJobManager, if (s_logger.isDebugEnabled()) { s_logger.debug("Cancel left-over job-" + job.getId()); } + cleanupResources(job); job.setStatus(JobInfo.Status.FAILED); job.setResultCode(ApiErrorCode.INTERNAL_ERROR.getHttpCode()); job.setResult("job cancelled because of management server restart or shutdown"); @@ -1101,26 +1124,8 @@ public class AsyncJobManagerImpl extends ManagerBase implements AsyncJobManager, s_logger.debug("Purge queue item for cancelled job-" + job.getId()); } _queueMgr.purgeAsyncJobQueueItemId(job.getId()); - if (ApiCommandResourceType.Volume.toString().equals(job.getInstanceType())) { - - try { - _volumeDetailsDao.removeDetail(job.getInstanceId(), "SNAPSHOT_ID"); - _volsDao.remove(job.getInstanceId()); - } catch (Exception e) { - s_logger.error("Unexpected exception while removing concurrent request meta data :" + e.getLocalizedMessage()); - } - } - } - final List snapshotList = _snapshotDetailsDao.findDetails(AsyncJob.Constants.MS_ID, Long.toString(msid), false); - for (final SnapshotDetailsVO snapshotDetailsVO : snapshotList) { - SnapshotInfo snapshot = snapshotFactory.getSnapshotOnPrimaryStore(snapshotDetailsVO.getResourceId()); - if (snapshot == null) { - _snapshotDetailsDao.remove(snapshotDetailsVO.getId()); - continue; - } - snapshotSrv.processEventOnSnapshotObject(snapshot, Snapshot.Event.OperationFailed); - _snapshotDetailsDao.removeDetail(snapshotDetailsVO.getResourceId(), AsyncJob.Constants.MS_ID); } + cleanupFailedSnapshotsCreatedWithDefaultStrategy(msid); } }); } catch (Throwable e) { @@ -1128,6 +1133,106 @@ public class AsyncJobManagerImpl extends ManagerBase implements AsyncJobManager, } } + /* + Cleanup Resources in transition state and move them to appropriate state + This will allow other operation on the resource, instead of being stuck in transition state + */ + protected boolean cleanupResources(AsyncJobVO job) { + try { + ApiCommandResourceType resourceType = ApiCommandResourceType.fromString(job.getInstanceType()); + if (resourceType == null) { + s_logger.warn("Unknown ResourceType. Skip Cleanup: " + job.getInstanceType()); + return true; + } + switch (resourceType) { + case Volume: + return cleanupVolume(job.getInstanceId()); + case VirtualMachine: + return cleanupVirtualMachine(job.getInstanceId()); + case Network: + return cleanupNetwork(job.getInstanceId()); + } + } catch (Exception e) { + s_logger.warn("Error while cleaning up resource: [" + job.getInstanceType().toString() + "] with Id: " + job.getInstanceId(), e); + return false; + } + return true; + } + + private boolean cleanupVolume(final long volumeId) { + VolumeInfo vol = volFactory.getVolume(volumeId); + if (vol == null) { + s_logger.warn("Volume not found. Skip Cleanup. VolumeId: " + volumeId); + return true; + } + if (vol.getState().isTransitional()) { + s_logger.debug("Cleaning up volume with Id: " + volumeId); + boolean status = vol.stateTransit(Volume.Event.OperationFailed); + cleanupFailedVolumesCreatedFromSnapshots(volumeId); + return status; + } + s_logger.debug("Volume not in transition state. Skip cleanup. VolumeId: " + volumeId); + return true; + } + + private boolean cleanupVirtualMachine(final long vmId) throws Exception { + VMInstanceVO vmInstanceVO = _vmInstanceDao.findById(vmId); + if (vmInstanceVO == null) { + s_logger.warn("Instance not found. Skip Cleanup. InstanceId: " + vmId); + return true; + } + if (vmInstanceVO.getState().isTransitional()) { + s_logger.debug("Cleaning up Instance with Id: " + vmId); + return virtualMachineManager.stateTransitTo(vmInstanceVO, VirtualMachine.Event.OperationFailed, vmInstanceVO.getHostId()); + } + s_logger.debug("Instance not in transition state. Skip cleanup. InstanceId: " + vmId); + return true; + } + + private boolean cleanupNetwork(final long networkId) throws Exception { + NetworkVO networkVO = networkDao.findById(networkId); + if (networkVO == null) { + s_logger.warn("Network not found. Skip Cleanup. NetworkId: " + networkId); + return true; + } + if (Network.State.Implementing.equals(networkVO.getState())) { + try { + s_logger.debug("Cleaning up Network with Id: " + networkId); + return networkOrchestrationService.stateTransitTo(networkVO, Network.Event.OperationFailed); + } catch (final NoTransitionException e) { + networkVO.setState(Network.State.Shutdown); + networkDao.update(networkVO.getId(), networkVO); + } + } + s_logger.debug("Network not in transition state. Skip cleanup. NetworkId: " + networkId); + return true; + } + + private void cleanupFailedVolumesCreatedFromSnapshots(final long volumeId) { + try { + VolumeDetailVO volumeDetail = _volumeDetailsDao.findDetail(volumeId, VolumeService.SNAPSHOT_ID); + if (volumeDetail != null) { + _volumeDetailsDao.removeDetail(volumeId, VolumeService.SNAPSHOT_ID); + _volsDao.remove(volumeId); + } + } catch (Exception e) { + s_logger.error("Unexpected exception while removing concurrent request meta data :" + e.getLocalizedMessage()); + } + } + + private void cleanupFailedSnapshotsCreatedWithDefaultStrategy(final long msid) { + final List snapshotList = _snapshotDetailsDao.findDetails(AsyncJob.Constants.MS_ID, Long.toString(msid), false); + for (final SnapshotDetailsVO snapshotDetailsVO : snapshotList) { + SnapshotInfo snapshot = snapshotFactory.getSnapshotOnPrimaryStore(snapshotDetailsVO.getResourceId()); + if (snapshot == null) { + _snapshotDetailsDao.remove(snapshotDetailsVO.getId()); + continue; + } + snapshotSrv.processEventOnSnapshotObject(snapshot, Snapshot.Event.OperationFailed); + _snapshotDetailsDao.removeDetail(snapshotDetailsVO.getResourceId(), AsyncJob.Constants.MS_ID); + } + } + @Override public void onManagementNodeJoined(List nodeList, long selfNodeId) { } diff --git a/framework/jobs/src/test/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImplTest.java b/framework/jobs/src/test/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImplTest.java new file mode 100644 index 00000000000..0be5dbc01cb --- /dev/null +++ b/framework/jobs/src/test/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImplTest.java @@ -0,0 +1,96 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.framework.jobs.impl; + +import com.cloud.network.Network; +import com.cloud.network.dao.NetworkDao; +import com.cloud.network.dao.NetworkVO; +import com.cloud.storage.Volume; +import com.cloud.utils.fsm.NoTransitionException; +import com.cloud.vm.VMInstanceVO; +import com.cloud.vm.VirtualMachine; +import com.cloud.vm.VirtualMachineManager; +import com.cloud.vm.dao.VMInstanceDao; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; + +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class AsyncJobManagerImplTest { + @Spy + @InjectMocks + AsyncJobManagerImpl asyncJobManager; + @Mock + VolumeDataFactory volFactory; + @Mock + VMInstanceDao vmInstanceDao; + @Mock + VirtualMachineManager virtualMachineManager; + @Mock + NetworkDao networkDao; + @Mock + NetworkOrchestrationService networkOrchestrationService; + + @Test + public void testCleanupVolumeResource() { + AsyncJobVO job = new AsyncJobVO(); + job.setInstanceType(ApiCommandResourceType.Volume.toString()); + job.setInstanceId(1L); + VolumeInfo volumeInfo = Mockito.mock(VolumeInfo.class); + when(volFactory.getVolume(Mockito.anyLong())).thenReturn(volumeInfo); + when(volumeInfo.getState()).thenReturn(Volume.State.Attaching); + asyncJobManager.cleanupResources(job); + Mockito.verify(volumeInfo, Mockito.times(1)).stateTransit(Volume.Event.OperationFailed); + } + + @Test + public void testCleanupVmResource() throws NoTransitionException { + AsyncJobVO job = new AsyncJobVO(); + job.setInstanceType(ApiCommandResourceType.VirtualMachine.toString()); + job.setInstanceId(1L); + VMInstanceVO vmInstanceVO = Mockito.mock(VMInstanceVO.class); + when(vmInstanceDao.findById(Mockito.anyLong())).thenReturn(vmInstanceVO); + when(vmInstanceVO.getState()).thenReturn(VirtualMachine.State.Starting); + when(vmInstanceVO.getHostId()).thenReturn(1L); + asyncJobManager.cleanupResources(job); + Mockito.verify(virtualMachineManager, Mockito.times(1)).stateTransitTo(vmInstanceVO, VirtualMachine.Event.OperationFailed, 1L); + } + + @Test + public void testCleanupNetworkResource() throws NoTransitionException { + AsyncJobVO job = new AsyncJobVO(); + job.setInstanceType(ApiCommandResourceType.Network.toString()); + job.setInstanceId(1L); + NetworkVO networkVO = Mockito.mock(NetworkVO.class); + when(networkDao.findById(Mockito.anyLong())).thenReturn(networkVO); + when(networkVO.getState()).thenReturn(Network.State.Implementing); + asyncJobManager.cleanupResources(job); + Mockito.verify(networkOrchestrationService, Mockito.times(1)).stateTransitTo(networkVO, + Network.Event.OperationFailed); + } +} diff --git a/server/src/test/java/com/cloud/vpc/MockNetworkManagerImpl.java b/server/src/test/java/com/cloud/vpc/MockNetworkManagerImpl.java index 04556c49ab2..288211c4330 100644 --- a/server/src/test/java/com/cloud/vpc/MockNetworkManagerImpl.java +++ b/server/src/test/java/com/cloud/vpc/MockNetworkManagerImpl.java @@ -26,6 +26,7 @@ import javax.naming.ConfigurationException; import com.cloud.dc.DataCenter; import com.cloud.network.PublicIpQuarantine; +import com.cloud.utils.fsm.NoTransitionException; import org.apache.cloudstack.acl.ControlledEntity.ACLType; import org.apache.cloudstack.api.command.admin.address.ReleasePodIpCmdByAdmin; import org.apache.cloudstack.api.command.admin.network.DedicateGuestVlanRangeCmd; @@ -824,6 +825,11 @@ public class MockNetworkManagerImpl extends ManagerBase implements NetworkOrches return null; } + @Override + public boolean stateTransitTo(Network network, Network.Event e) throws NoTransitionException { + return true; + } + @Override public boolean isNetworkInlineMode(Network network) { // TODO Auto-generated method stub