diff --git a/core/src/main/java/org/apache/cloudstack/storage/command/DettachCommand.java b/core/src/main/java/org/apache/cloudstack/storage/command/DettachCommand.java index eeeadaac42b..ff7e1f1f64e 100644 --- a/core/src/main/java/org/apache/cloudstack/storage/command/DettachCommand.java +++ b/core/src/main/java/org/apache/cloudstack/storage/command/DettachCommand.java @@ -32,6 +32,7 @@ public class DettachCommand extends StorageSubSystemCommand { private int _storagePort; private Map params; private boolean forced; + private long waitDetachDevice; public DettachCommand(final DiskTO disk, final String vmName) { super(); @@ -115,6 +116,14 @@ public class DettachCommand extends StorageSubSystemCommand { this.forced = forced; } + public void setWaitDetachDevice(long wait) { + this.waitDetachDevice = wait; + } + + public long getWaitDetachDevice(){ + return waitDetachDevice; + } + @Override public void setExecuteInSequence(final boolean inSeq) { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java index 6969faa1cfa..151e652e251 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java @@ -25,7 +25,6 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; -import java.net.URISyntaxException; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.Arrays; @@ -155,6 +154,10 @@ public class KVMStorageProcessor implements StorageProcessor { private static final String CEPH_AUTH_KEY = "key"; private static final String CEPH_CLIENT_MOUNT_TIMEOUT = "client_mount_timeout"; private static final String CEPH_DEFAULT_MOUNT_TIMEOUT = "30"; + /** + * Time interval before rechecking virsh commands + */ + private long waitDelayForVirshCommands = 1000l; public KVMStorageProcessor(final KVMStoragePoolManager storagePoolMgr, final LibvirtComputingResource resource) { this.storagePoolMgr = storagePoolMgr; @@ -1068,10 +1071,9 @@ public class KVMStorageProcessor implements StorageProcessor { } } } - - protected synchronized String attachOrDetachISO(final Connect conn, final String vmName, String isoPath, final boolean isAttach, Map params) throws LibvirtException, URISyntaxException, - InternalErrorException { - String isoXml = null; + protected synchronized void attachOrDetachISO(final Connect conn, final String vmName, String isoPath, final boolean isAttach, Map params) throws + LibvirtException, InternalErrorException { + DiskDef iso = new DiskDef(); boolean isUefiEnabled = MapUtils.isNotEmpty(params) && params.containsKey("UEFI"); if (isoPath != null && isAttach) { final int index = isoPath.lastIndexOf("/"); @@ -1081,18 +1083,14 @@ public class KVMStorageProcessor implements StorageProcessor { final KVMPhysicalDisk isoVol = secondaryPool.getPhysicalDisk(name); isoPath = isoVol.getPath(); - final DiskDef iso = new DiskDef(); iso.defISODisk(isoPath, isUefiEnabled); - isoXml = iso.toString(); } else { - final DiskDef iso = new DiskDef(); iso.defISODisk(null, isUefiEnabled); - isoXml = iso.toString(); } final List disks = resource.getDisks(conn, vmName); - final String result = attachOrDetachDevice(conn, true, vmName, isoXml); - if (result == null && !isAttach) { + attachOrDetachDevice(conn, true, vmName, iso); + if (!isAttach) { for (final DiskDef disk : disks) { if (disk.getDeviceType() == DiskDef.DeviceType.CDROM) { resource.cleanupDisk(disk); @@ -1100,7 +1098,6 @@ public class KVMStorageProcessor implements StorageProcessor { } } - return result; } @Override @@ -1115,8 +1112,6 @@ public class KVMStorageProcessor implements StorageProcessor { attachOrDetachISO(conn, cmd.getVmName(), dataStoreUrl + File.separator + isoTO.getPath(), true, cmd.getControllerInfo()); } catch (final LibvirtException e) { return new Answer(cmd, false, e.toString()); - } catch (final URISyntaxException e) { - return new Answer(cmd, false, e.toString()); } catch (final InternalErrorException e) { return new Answer(cmd, false, e.toString()); } catch (final InvalidParameterValueException e) { @@ -1138,8 +1133,6 @@ public class KVMStorageProcessor implements StorageProcessor { attachOrDetachISO(conn, cmd.getVmName(), dataStoreUrl + File.separator + isoTO.getPath(), false, cmd.getParams()); } catch (final LibvirtException e) { return new Answer(cmd, false, e.toString()); - } catch (final URISyntaxException e) { - return new Answer(cmd, false, e.toString()); } catch (final InternalErrorException e) { return new Answer(cmd, false, e.toString()); } catch (final InvalidParameterValueException e) { @@ -1169,27 +1162,47 @@ public class KVMStorageProcessor implements StorageProcessor { } return store.getUrl(); } + protected synchronized void attachOrDetachDevice(final Connect conn, final boolean attach, final String vmName, final DiskDef xml) + throws LibvirtException, InternalErrorException { + attachOrDetachDevice(conn, attach, vmName, xml, 0l); + } - protected synchronized String attachOrDetachDevice(final Connect conn, final boolean attach, final String vmName, final String xml) throws LibvirtException, InternalErrorException { + /** + * Attaches or detaches a device (ISO or disk) to an instance. + * @param conn libvirt connection + * @param attach boolean that determines whether the device will be attached or detached + * @param vmName instance name + * @param diskDef disk definition or iso to be attached or detached + * @param waitDetachDevice value set in milliseconds to wait before assuming device removal failed + * @throws LibvirtException + * @throws InternalErrorException + */ + protected synchronized void attachOrDetachDevice(final Connect conn, final boolean attach, final String vmName, final DiskDef diskDef, long waitDetachDevice) + throws LibvirtException, InternalErrorException { Domain dm = null; + String diskXml = diskDef.toString(); + String diskPath = diskDef.getDiskPath(); try { dm = conn.domainLookupByName(vmName); if (attach) { - s_logger.debug("Attaching device: " + xml); - dm.attachDevice(xml); - } else { - s_logger.debug("Detaching device: " + xml); - dm.detachDevice(xml); - LibvirtDomainXMLParser parser = new LibvirtDomainXMLParser(); - parser.parseDomainXML(dm.getXMLDesc(0)); - List disks = parser.getDisks(); - for (DiskDef diskDef : disks) { - if (StringUtils.contains(xml, diskDef.getDiskPath())) { - throw new InternalErrorException("Could not detach volume. Probably the VM is in boot state at the moment"); - } - } + s_logger.debug("Attaching device: " + diskXml); + dm.attachDevice(diskXml); + return; } + s_logger.debug(String.format("Detaching device: [%s].", diskXml)); + dm.detachDevice(diskXml); + long wait = waitDetachDevice; + while (!checkDetachSuccess(diskPath, dm) && wait > 0) { + wait = getWaitAfterSleep(dm, diskPath, wait); + } + if (wait <= 0) { + throw new InternalErrorException(String.format("Could not detach volume after sending the command and waiting for [%s] milliseconds. Probably the VM does " + + "not support the sent detach command or the device is busy at the moment. Try again in a couple of minutes.", + waitDetachDevice)); + } + s_logger.debug(String.format("The detach command was executed successfully. The device [%s] was removed from the VM instance with UUID [%s].", + diskPath, dm.getUUIDString())); } catch (final LibvirtException e) { if (attach) { s_logger.warn("Failed to attach device to " + vmName + ": " + e.getMessage()); @@ -1206,15 +1219,115 @@ public class KVMStorageProcessor implements StorageProcessor { } } } - - return null; } - protected synchronized String attachOrDetachDisk(final Connect conn, final boolean attach, final String vmName, final KVMPhysicalDisk attachingDisk, final int devId, final String serial, - final Long bytesReadRate, final Long bytesReadRateMax, final Long bytesReadRateMaxLength, - final Long bytesWriteRate, final Long bytesWriteRateMax, final Long bytesWriteRateMaxLength, - final Long iopsReadRate, final Long iopsReadRateMax, final Long iopsReadRateMaxLength, - final Long iopsWriteRate, final Long iopsWriteRateMax, final Long iopsWriteRateMaxLength, final String cacheMode, final DiskDef.LibvirtDiskEncryptDetails encryptDetails) throws LibvirtException, InternalErrorException { + /** + * Waits {@link #waitDelayForVirshCommands} milliseconds before checking again if the device has been removed. + * @return The configured value in wait.detach.device reduced by {@link #waitDelayForVirshCommands} + * @throws LibvirtException + */ + private long getWaitAfterSleep(Domain dm, String diskPath, long wait) throws LibvirtException { + try { + wait -= waitDelayForVirshCommands; + Thread.sleep(waitDelayForVirshCommands); + s_logger.trace(String.format("Trying to detach device [%s] from VM instance with UUID [%s]. " + + "Waiting [%s] milliseconds before assuming the VM was unable to detach the volume.", diskPath, dm.getUUIDString(), wait)); + } catch (InterruptedException e) { + throw new CloudRuntimeException(e); + } + return wait; + } + + /** + * Checks if the device has been removed from the instance + * @param diskPath Path to the device that was removed + * @param dm instance to be checked if the device was properly removed + * @throws LibvirtException + */ + protected boolean checkDetachSuccess(String diskPath, Domain dm) throws LibvirtException { + LibvirtDomainXMLParser parser = new LibvirtDomainXMLParser(); + parser.parseDomainXML(dm.getXMLDesc(0)); + List disks = parser.getDisks(); + for (DiskDef diskDef : disks) { + if (StringUtils.equals(diskPath, diskDef.getDiskPath())) { + s_logger.debug(String.format("The hypervisor sent the detach command, but it is still possible to identify the device [%s] in the instance with UUID [%s].", + diskPath, dm.getUUIDString())); + return false; + } + } + return true; + } + + /** + * Attaches or detaches a disk to an instance. + * @param conn libvirt connection + * @param attach boolean that determines whether the device will be attached or detached + * @param vmName instance name + * @param attachingDisk kvm physical disk + * @param devId device id in instance + * @param serial + * @param bytesReadRate bytes read rate + * @param bytesReadRateMax bytes read rate max + * @param bytesReadRateMaxLength bytes read rate max length + * @param bytesWriteRate bytes write rate + * @param bytesWriteRateMax bytes write rate amx + * @param bytesWriteRateMaxLength bytes write rate max length + * @param iopsReadRate iops read rate + * @param iopsReadRateMax iops read rate max + * @param iopsReadRateMaxLength iops read rate max length + * @param iopsWriteRate iops write rate + * @param iopsWriteRateMax iops write rate max + * @param iopsWriteRateMaxLength iops write rate max length + * @param cacheMode cache mode + * @param encryptDetails encrypt details + * @throws LibvirtException + * @throws InternalErrorException + */ + protected synchronized void attachOrDetachDisk(final Connect conn, final boolean attach, final String vmName, final KVMPhysicalDisk attachingDisk, final int devId, + final String serial, final Long bytesReadRate, final Long bytesReadRateMax, final Long bytesReadRateMaxLength, + final Long bytesWriteRate, final Long bytesWriteRateMax, final Long bytesWriteRateMaxLength, final Long iopsReadRate, + final Long iopsReadRateMax, final Long iopsReadRateMaxLength, final Long iopsWriteRate, final Long iopsWriteRateMax, + final Long iopsWriteRateMaxLength, final String cacheMode, final DiskDef.LibvirtDiskEncryptDetails encryptDetails) + throws LibvirtException, InternalErrorException { + attachOrDetachDisk(conn, attach, vmName, attachingDisk, devId, serial, bytesReadRate, bytesReadRateMax, bytesReadRateMaxLength, + bytesWriteRate, bytesWriteRateMax, bytesWriteRateMaxLength, iopsReadRate, iopsReadRateMax, iopsReadRateMaxLength, iopsWriteRate, + iopsWriteRateMax, iopsWriteRateMaxLength, cacheMode, encryptDetails, 0l); + } + + /** + * + * Attaches or detaches a disk to an instance. + * @param conn libvirt connection + * @param attach boolean that determines whether the device will be attached or detached + * @param vmName instance name + * @param attachingDisk kvm physical disk + * @param devId device id in instance + * @param serial + * @param bytesReadRate bytes read rate + * @param bytesReadRateMax bytes read rate max + * @param bytesReadRateMaxLength bytes read rate max length + * @param bytesWriteRate bytes write rate + * @param bytesWriteRateMax bytes write rate amx + * @param bytesWriteRateMaxLength bytes write rate max length + * @param iopsReadRate iops read rate + * @param iopsReadRateMax iops read rate max + * @param iopsReadRateMaxLength iops read rate max length + * @param iopsWriteRate iops write rate + * @param iopsWriteRateMax iops write rate max + * @param iopsWriteRateMaxLength iops write rate max length + * @param cacheMode cache mode + * @param encryptDetails encrypt details + * @param waitDetachDevice value set in milliseconds to wait before assuming device removal failed + * @throws LibvirtException + * @throws InternalErrorException + */ + protected synchronized void attachOrDetachDisk(final Connect conn, final boolean attach, final String vmName, final KVMPhysicalDisk attachingDisk, final int devId, + final String serial, final Long bytesReadRate, final Long bytesReadRateMax, final Long bytesReadRateMaxLength, + final Long bytesWriteRate, final Long bytesWriteRateMax, final Long bytesWriteRateMaxLength, final Long iopsReadRate, + final Long iopsReadRateMax, final Long iopsReadRateMaxLength, final Long iopsWriteRate, final Long iopsWriteRateMax, + final Long iopsWriteRateMaxLength, final String cacheMode, final DiskDef.LibvirtDiskEncryptDetails encryptDetails, + long waitDetachDevice) + throws LibvirtException, InternalErrorException { List disks = null; Domain dm = null; DiskDef diskdef = null; @@ -1244,7 +1357,9 @@ public class KVMStorageProcessor implements StorageProcessor { } } if (diskdef == null) { - throw new InternalErrorException("disk: " + attachingDisk.getPath() + " is not attached before"); + s_logger.warn(String.format("Could not find disk [%s] attached to VM instance with UUID [%s]. We will set it as detached in the database to ensure consistency.", + attachingDisk.getPath(), dm.getUUIDString())); + return; } } else { DiskDef.DiskBus busT = DiskDef.DiskBus.VIRTIO; @@ -1338,8 +1453,7 @@ public class KVMStorageProcessor implements StorageProcessor { } } - final String xml = diskdef.toString(); - return attachOrDetachDevice(conn, attach, vmName, xml); + attachOrDetachDevice(conn, attach, vmName, diskdef, waitDetachDevice); } finally { if (dm != null) { dm.free(); @@ -1399,6 +1513,7 @@ public class KVMStorageProcessor implements StorageProcessor { final PrimaryDataStoreTO primaryStore = (PrimaryDataStoreTO)vol.getDataStore(); final String vmName = cmd.getVmName(); final String serial = resource.diskUuidToSerial(vol.getUuid()); + long waitDetachDevice = cmd.getWaitDetachDevice(); try { final Connect conn = LibvirtConnection.getConnectionByVmName(vmName); @@ -1409,7 +1524,7 @@ public class KVMStorageProcessor implements StorageProcessor { vol.getBytesReadRate(), vol.getBytesReadRateMax(), vol.getBytesReadRateMaxLength(), vol.getBytesWriteRate(), vol.getBytesWriteRateMax(), vol.getBytesWriteRateMaxLength(), vol.getIopsReadRate(), vol.getIopsReadRateMax(), vol.getIopsReadRateMaxLength(), - vol.getIopsWriteRate(), vol.getIopsWriteRateMax(), vol.getIopsWriteRateMaxLength(), volCacheMode, null); + vol.getIopsWriteRate(), vol.getIopsWriteRateMax(), vol.getIopsWriteRateMaxLength(), volCacheMode, null, waitDetachDevice); storagePoolMgr.disconnectPhysicalDisk(primaryStore.getPoolType(), primaryStore.getUuid(), vol.getPath()); diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessorTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessorTest.java index 9f6a46a3c7e..98a5b130bae 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessorTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessorTest.java @@ -18,7 +18,9 @@ */ package com.cloud.hypervisor.kvm.storage; +import com.cloud.exception.InternalErrorException; import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.hypervisor.kvm.resource.LibvirtDomainXMLParser; import com.cloud.hypervisor.kvm.resource.LibvirtVMDef; import com.cloud.hypervisor.kvm.resource.wrapper.LibvirtUtilitiesHelper; import com.cloud.storage.template.TemplateConstants; @@ -51,10 +53,12 @@ import org.mockito.Mockito; import org.mockito.MockitoAnnotations; import org.mockito.Spy; import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PowerMockIgnore; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; @PrepareForTest({ Script.class }) +@PowerMockIgnore({"javax.xml.*", "org.xml.*", "org.w3c.dom.*"}) @RunWith(PowerMockRunner.class) public class KVMStorageProcessorTest { @@ -87,6 +91,12 @@ public class KVMStorageProcessorTest { @Mock Connect connectMock; + @Mock + LibvirtDomainXMLParser libvirtDomainXMLParserMock; + @Mock + LibvirtVMDef.DiskDef diskDefMock; + + private static final String directDownloadTemporaryPath = "/var/lib/libvirt/images/dd"; private static final long templateSize = 80000L; @@ -348,4 +358,88 @@ public class KVMStorageProcessorTest { storageProcessorSpy.deleteSnapshotFile(snapshotObjectToMock); } + + private void checkDetachSucessTest(boolean duplicate) throws Exception { + List disks = createDiskDefs(2, duplicate); + PowerMockito.when(domainMock.getXMLDesc(Mockito.anyInt())).thenReturn("test"); + PowerMockito.whenNew(LibvirtDomainXMLParser.class).withAnyArguments().thenReturn(libvirtDomainXMLParserMock); + PowerMockito.when(libvirtDomainXMLParserMock.parseDomainXML(Mockito.anyString())).thenReturn(true); + PowerMockito.when(libvirtDomainXMLParserMock.getDisks()).thenReturn(disks); + } + + @Test + @PrepareForTest(KVMStorageProcessor.class) + public void checkDetachSucessTestDetachReturnTrue() throws Exception { + checkDetachSucessTest(false); + Assert.assertTrue(storageProcessorSpy.checkDetachSuccess("path", domainMock)); + } + + @Test + @PrepareForTest(KVMStorageProcessor.class) + public void checkDetachSucessTestDetachReturnFalse() throws Exception { + checkDetachSucessTest(true); + Assert.assertFalse(storageProcessorSpy.checkDetachSuccess("path", domainMock)); + } + + private void attachOrDetachDeviceTest (boolean attach, String vmName, LibvirtVMDef.DiskDef xml) throws LibvirtException, InternalErrorException { + storageProcessorSpy.attachOrDetachDevice(connectMock, attach, vmName, xml); + } + @PrepareForTest(KVMStorageProcessor.class) + private void attachOrDetachDeviceTest (boolean attach, String vmName, LibvirtVMDef.DiskDef xml, long waitDetachDevice) throws LibvirtException, InternalErrorException { + storageProcessorSpy.attachOrDetachDevice(connectMock, attach, vmName, xml, waitDetachDevice); + } + + @Test (expected = LibvirtException.class) + @PrepareForTest(KVMStorageProcessor.class) + public void attachOrDetachDeviceTestThrowLibvirtException() throws LibvirtException, InternalErrorException { + Mockito.when(connectMock.domainLookupByName(Mockito.anyString())).thenThrow(LibvirtException.class); + attachOrDetachDeviceTest(true, "vmName", diskDefMock); + } + + @PrepareForTest(KVMStorageProcessor.class) + public void attachOrDetachDeviceTestAttachSuccess() throws LibvirtException, InternalErrorException { + Mockito.when(connectMock.domainLookupByName("vmName")).thenReturn(domainMock); + attachOrDetachDeviceTest(true, "vmName", diskDefMock); + Mockito.verify(domainMock, Mockito.times(1)).attachDevice(Mockito.anyString()); + } + + @Test (expected = LibvirtException.class) + @PrepareForTest(KVMStorageProcessor.class) + public void attachOrDetachDeviceTestAttachThrowLibvirtException() throws LibvirtException, InternalErrorException { + Mockito.when(connectMock.domainLookupByName("vmName")).thenReturn(domainMock); + Mockito.when(diskDefMock.toString()).thenReturn("diskDef"); + Mockito.when(diskDefMock.getDiskPath()).thenReturn("diskDef"); + Mockito.doThrow(LibvirtException.class).when(domainMock).attachDevice(Mockito.anyString()); + attachOrDetachDeviceTest(true, "vmName", diskDefMock); + } + + @Test (expected = LibvirtException.class) + @PrepareForTest(KVMStorageProcessor.class) + public void attachOrDetachDeviceTestDetachThrowLibvirtException() throws LibvirtException, InternalErrorException { + Mockito.when(connectMock.domainLookupByName("vmName")).thenReturn(domainMock); + Mockito.doThrow(LibvirtException.class).when(domainMock).detachDevice(Mockito.anyString()); + attachOrDetachDeviceTest(false, "vmName", diskDefMock); + } + + @Test + @PrepareForTest(KVMStorageProcessor.class) + public void attachOrDetachDeviceTestDetachSuccess() throws LibvirtException, InternalErrorException { + Mockito.when(connectMock.domainLookupByName("vmName")).thenReturn(domainMock); + PowerMockito.doReturn(true).when(storageProcessorSpy).checkDetachSuccess(Mockito.anyString(), Mockito.any(Domain.class)); + Mockito.when(diskDefMock.toString()).thenReturn("diskDef"); + Mockito.when(diskDefMock.getDiskPath()).thenReturn("diskDef"); + attachOrDetachDeviceTest( false, "vmName", diskDefMock, 10000); + Mockito.verify(domainMock, Mockito.times(1)).detachDevice(Mockito.anyString()); + } + + @Test (expected = InternalErrorException.class) + @PrepareForTest(KVMStorageProcessor.class) + public void attachOrDetachDeviceTestDetachThrowInternalErrorException() throws LibvirtException, InternalErrorException { + Mockito.when(connectMock.domainLookupByName("vmName")).thenReturn(domainMock); + PowerMockito.doReturn(false).when(storageProcessorSpy).checkDetachSuccess(Mockito.anyString(), Mockito.any(Domain.class)); + Mockito.when(diskDefMock.toString()).thenReturn("diskDef"); + Mockito.when(diskDefMock.getDiskPath()).thenReturn("diskDef"); + attachOrDetachDeviceTest( false, "vmName", diskDefMock); + Mockito.verify(domainMock, Mockito.times(1)).detachDevice(Mockito.anyString()); + } } diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java index 7f5769aa785..86849a87a91 100644 --- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java @@ -337,6 +337,14 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic public static final ConfigKey MatchStoragePoolTagsWithDiskOffering = new ConfigKey("Advanced", Boolean.class, "match.storage.pool.tags.with.disk.offering", "true", "If true, volume's disk offering can be changed only with the matched storage tags", true, ConfigKey.Scope.Zone); + public static final ConfigKey WaitDetachDevice = new ConfigKey<>( + "Advanced", + Long.class, + "wait.detach.device", + "10000", + "Time (in milliseconds) to wait before assuming the VM was unable to detach a volume after the hypervisor sends the detach command.", + true); + private final StateMachine2 _volStateMachine; private static final Set STATES_VOLUME_CANNOT_BE_DESTROYED = new HashSet<>(Arrays.asList(Volume.State.Destroy, Volume.State.Expunging, Volume.State.Expunged, Volume.State.Allocated)); @@ -2777,6 +2785,7 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic cmd.setStoragePort(volumePool.getPort()); cmd.set_iScsiName(volume.get_iScsiName()); + cmd.setWaitDetachDevice(WaitDetachDevice.value()); try { answer = _agentMgr.send(hostId, cmd); @@ -4528,6 +4537,12 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic @Override public ConfigKey[] getConfigKeys() { - return new ConfigKey[] {ConcurrentMigrationsThresholdPerDatastore, AllowUserExpungeRecoverVolume, MatchStoragePoolTagsWithDiskOffering, UseHttpsToUpload}; + return new ConfigKey[] { + ConcurrentMigrationsThresholdPerDatastore, + AllowUserExpungeRecoverVolume, + MatchStoragePoolTagsWithDiskOffering, + UseHttpsToUpload, + WaitDetachDevice + }; } }