From b5f7e307a11e0fb7305b94f0e0795f89b7abbb7d Mon Sep 17 00:00:00 2001 From: Edison Su Date: Wed, 25 Sep 2013 16:01:45 -0700 Subject: [PATCH] CLOUDSTACK-4456: CLOUDSTACK-4457: CLOUDSTACK-4459: harden kvm getvolume. It's possible that one volume created on other kvm host, won't show up on another host, try more times to refresh storage pool if volume won't shown up Conflicts: engine/storage/integration-test/test/org/apache/cloudstack/storage/test/FakeDriverTestConfiguration.java plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java --- .../test/FakeDriverTestConfiguration.java | 5 +- .../test/resource/fakeDriverTestContext.xml | 2 + .../datastore/PrimaryDataStoreImpl.java | 6 +++ .../resource/LibvirtComputingResource.java | 19 ++++--- .../kvm/storage/KVMStoragePoolManager.java | 33 ++++++++++++ .../kvm/storage/KVMStorageProcessor.java | 53 ++++++------------- 6 files changed, 70 insertions(+), 48 deletions(-) diff --git a/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/FakeDriverTestConfiguration.java b/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/FakeDriverTestConfiguration.java index 943b73cb97e..5e3e8851066 100644 --- a/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/FakeDriverTestConfiguration.java +++ b/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/FakeDriverTestConfiguration.java @@ -24,6 +24,7 @@ import com.cloud.storage.snapshot.SnapshotSchedulerImpl; import com.cloud.user.DomainManager; import com.cloud.utils.component.ComponentContext; + import org.apache.cloudstack.engine.subsystem.api.storage.DataMotionStrategy; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProvider; import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector; @@ -50,7 +51,6 @@ public class FakeDriverTestConfiguration extends ChildTestConfiguration{ return strategy; } - @Bean public SnapshotScheduler SnapshotScheduler() { return Mockito.mock(SnapshotScheduler.class); @@ -61,10 +61,11 @@ public class FakeDriverTestConfiguration extends ChildTestConfiguration{ return Mockito.mock(DomainManager.class); } + @Override @Bean public EndPointSelector selector() { return ComponentContext.inject(DefaultEndPointSelector.class); } - } + diff --git a/engine/storage/integration-test/test/resource/fakeDriverTestContext.xml b/engine/storage/integration-test/test/resource/fakeDriverTestContext.xml index 98209de67e9..c3c8ef5a6bb 100644 --- a/engine/storage/integration-test/test/resource/fakeDriverTestContext.xml +++ b/engine/storage/integration-test/test/resource/fakeDriverTestContext.xml @@ -85,4 +85,6 @@ + + diff --git a/engine/storage/volume/src/org/apache/cloudstack/storage/datastore/PrimaryDataStoreImpl.java b/engine/storage/volume/src/org/apache/cloudstack/storage/datastore/PrimaryDataStoreImpl.java index 7c8f49a9c9b..22ff2209b19 100644 --- a/engine/storage/volume/src/org/apache/cloudstack/storage/datastore/PrimaryDataStoreImpl.java +++ b/engine/storage/volume/src/org/apache/cloudstack/storage/datastore/PrimaryDataStoreImpl.java @@ -266,6 +266,12 @@ public class PrimaryDataStoreImpl implements PrimaryDataStore { } } else if (obj.getType() == DataObjectType.SNAPSHOT) { return objectInStoreMgr.create(obj, this); + } else if (obj.getType() == DataObjectType.VOLUME) { + VolumeVO vol = volumeDao.findById(obj.getId()); + if (vol != null) { + vol.setPoolId(this.getId()); + volumeDao.update(vol.getId(), vol); + } } return objectInStoreMgr.get(obj, this); diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 74f02c0c639..6a7a56251a1 100755 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -3627,10 +3627,10 @@ ServerResource { physicalDisk = secondaryStorage.getPhysicalDisk(volName); } else if (volume.getType() != Volume.Type.ISO) { PrimaryDataStoreTO store = (PrimaryDataStoreTO)data.getDataStore(); - pool = _storagePoolMgr.getStoragePool( - store.getPoolType(), - store.getUuid()); - physicalDisk = pool.getPhysicalDisk(data.getPath()); + physicalDisk = _storagePoolMgr.getPhysicalDisk( store.getPoolType(), + store.getUuid(), + data.getPath()); + pool = physicalDisk.getPool(); } String volPath = null; @@ -3703,10 +3703,9 @@ ServerResource { if (volume.getType() == Volume.Type.ROOT) { DataTO data = volume.getData(); PrimaryDataStoreTO store = (PrimaryDataStoreTO)data.getDataStore(); - KVMStoragePool pool = _storagePoolMgr.getStoragePool( - store.getPoolType(), - store.getUuid()); - KVMPhysicalDisk physicalDisk = pool.getPhysicalDisk(data.getPath()); + KVMPhysicalDisk physicalDisk = _storagePoolMgr.getPhysicalDisk( store.getPoolType(), + store.getUuid(), + data.getPath()); FilesystemDef rootFs = new FilesystemDef(physicalDisk.getPath(), "/"); vm.getDevices().addDevice(rootFs); break; @@ -3748,6 +3747,10 @@ ServerResource { // need to umount secondary storage String path = disk.getDiskPath(); String poolUuid = null; + if (path.endsWith("systemvm.iso")) { + //Don't need to clean up system vm iso, as it's stored in local + return true; + } if (path != null) { String[] token = path.split("/"); if (token.length > 3) { diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java index 40f4b5b785f..71bbc15e37c 100644 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java @@ -31,8 +31,11 @@ import com.cloud.hypervisor.kvm.resource.KVMHAMonitor; import com.cloud.storage.Storage.StoragePoolType; import com.cloud.storage.StorageLayer; import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.log4j.Logger; public class KVMStoragePoolManager { + private static final Logger s_logger = Logger + .getLogger(KVMStoragePoolManager.class); private class StoragePoolInformation { String name; String host; @@ -133,6 +136,36 @@ public class KVMStoragePoolManager { return createStoragePool(uuid, sourceHost, 0, sourcePath, "", protocol, false); } + public KVMPhysicalDisk getPhysicalDisk(StoragePoolType type, String poolUuid, String volName) { + int cnt = 0; + int retries = 10; + KVMPhysicalDisk vol = null; + //harden get volume, try cnt times to get volume, in case volume is created on other host + String errMsg = ""; + while (cnt < retries) { + try { + KVMStoragePool pool = getStoragePool(type, poolUuid); + vol = pool.getPhysicalDisk(volName); + if (vol != null) { + break; + } + + Thread.sleep(10000); + } catch (Exception e) { + s_logger.debug("Failed to find volume:" + volName + " due to" + e.toString() + ", retry:" + cnt); + errMsg = e.toString(); + } + cnt++; + } + + if (vol == null) { + throw new CloudRuntimeException(errMsg); + } else { + return vol; + } + + } + public KVMStoragePool createStoragePool( String name, String host, int port, String path, String userInfo, StoragePoolType type) { diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java index 3cca4fd087b..5ed93aab496 100644 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java @@ -289,8 +289,8 @@ public class KVMStorageProcessor implements StorageProcessor { if (primaryPool.getType() == StoragePoolType.CLVM) { vol = templateToPrimaryDownload(templatePath, primaryPool); } else { - BaseVol = primaryPool.getPhysicalDisk(templatePath); - vol = storagePoolMgr.createDiskFromTemplate(BaseVol, UUID.randomUUID().toString(), primaryPool); + BaseVol = storagePoolMgr.getPhysicalDisk(primaryStore.getPoolType(), primaryStore.getUuid(), templatePath); + vol = storagePoolMgr.createDiskFromTemplate(BaseVol, UUID.randomUUID().toString(), BaseVol.getPool()); } if (vol == null) { return new CopyCmdAnswer(" Can't create storage volume on storage pool"); @@ -394,28 +394,14 @@ public class KVMStorageProcessor implements StorageProcessor { String destVolumePath = destData.getPath(); String secondaryStorageUrl = nfsStore.getUrl(); KVMStoragePool secondaryStoragePool = null; - KVMStoragePool primaryPool = null; - try { - try { - primaryPool = storagePoolMgr.getStoragePool( - primaryStore.getPoolType(), - primaryStore.getUuid()); - } catch (CloudRuntimeException e) { - if (e.getMessage().contains("not found")) { - primaryPool = storagePoolMgr.createStoragePool(primaryStore.getUuid(), - primaryStore.getHost(), primaryStore.getPort(), - primaryStore.getPath(), null, - primaryStore.getPoolType()); - } else { - return new CopyCmdAnswer(e.getMessage()); - } - } + try { String volumeName = UUID.randomUUID().toString(); String destVolumeName = volumeName + "." + destFormat.getFileExtension(); - KVMPhysicalDisk volume = primaryPool.getPhysicalDisk(srcVolumePath); + KVMPhysicalDisk volume = storagePoolMgr.getPhysicalDisk(primaryStore.getPoolType(), primaryStore.getUuid(), srcVolumePath); volume.setFormat(PhysicalDiskFormat.valueOf(srcFormat.toString())); + secondaryStoragePool = storagePoolMgr.getStoragePoolByURI( secondaryStorageUrl); secondaryStoragePool.createFolder(destVolumePath); @@ -459,18 +445,9 @@ public class KVMStorageProcessor implements StorageProcessor { secondaryStorage = storagePoolMgr.getStoragePoolByURI(nfsImageStore.getUrl()); - try { - primary = storagePoolMgr.getStoragePool(primaryStore.getPoolType(), primaryStore.getUuid()); - } catch (CloudRuntimeException e) { - if (e.getMessage().contains("not found")) { - primary = storagePoolMgr.createStoragePool(primaryStore.getUuid(), primaryStore.getHost(), - primaryStore.getPort(), primaryStore.getPath(), null, primaryStore.getPoolType()); - } else { - return new CopyCmdAnswer(e.getMessage()); - } - } + primary = storagePoolMgr.getStoragePool(primaryStore.getPoolType(), primaryStore.getUuid()); - KVMPhysicalDisk disk = primary.getPhysicalDisk(volume.getPath()); + KVMPhysicalDisk disk = storagePoolMgr.getPhysicalDisk(primaryStore.getPoolType(), primaryStore.getUuid(), volume.getPath()); String tmpltPath = secondaryStorage.getLocalPath() + File.separator + templateFolder; this.storageLayer.mkdirs(tmpltPath); String templateName = UUID.randomUUID().toString(); @@ -661,9 +638,9 @@ public class KVMStorageProcessor implements StorageProcessor { snapshotRelPath = destSnapshot.getPath(); snapshotDestPath = ssPmountPath + File.separator + snapshotRelPath; - KVMStoragePool primaryPool = storagePoolMgr.getStoragePool(primaryStore.getPoolType(), - primaryStore.getUuid()); - KVMPhysicalDisk snapshotDisk = primaryPool.getPhysicalDisk(volumePath); + KVMPhysicalDisk snapshotDisk = storagePoolMgr.getPhysicalDisk(primaryStore.getPoolType(), + primaryStore.getUuid(), volumePath); + KVMStoragePool primaryPool = snapshotDisk.getPool(); /** * RBD snapshots can't be copied using qemu-img, so we have to use @@ -798,6 +775,7 @@ public class KVMStorageProcessor implements StorageProcessor { } } + protected synchronized String attachOrDetachISO(Connect conn, String vmName, String isoPath, boolean isAttach) throws LibvirtException, URISyntaxException, InternalErrorException { String isoXml = null; @@ -967,9 +945,8 @@ public class KVMStorageProcessor implements StorageProcessor { String vmName = cmd.getVmName(); try { Connect conn = LibvirtConnection.getConnectionByVmName(vmName); - KVMStoragePool primary = storagePoolMgr.getStoragePool(primaryStore.getPoolType(), primaryStore.getUuid()); + KVMPhysicalDisk phyDisk = storagePoolMgr.getPhysicalDisk(primaryStore.getPoolType(), primaryStore.getUuid(), vol.getPath()); - KVMPhysicalDisk phyDisk = primary.getPhysicalDisk(vol.getPath()); attachOrDetachDisk(conn, true, vmName, phyDisk, disk.getDiskSeq().intValue()); return new AttachAnswer(disk); @@ -990,9 +967,8 @@ public class KVMStorageProcessor implements StorageProcessor { String vmName = cmd.getVmName(); try { Connect conn = LibvirtConnection.getConnectionByVmName(vmName); - KVMStoragePool primary = storagePoolMgr.getStoragePool(primaryStore.getPoolType(), primaryStore.getUuid()); + KVMPhysicalDisk phyDisk = storagePoolMgr.getPhysicalDisk(primaryStore.getPoolType(), primaryStore.getUuid(), vol.getPath()); - KVMPhysicalDisk phyDisk = primary.getPhysicalDisk(vol.getPath()); attachOrDetachDisk(conn, false, vmName, phyDisk, disk.getDiskSeq().intValue()); return new DettachAnswer(disk); @@ -1064,7 +1040,8 @@ public class KVMStorageProcessor implements StorageProcessor { KVMStoragePool primaryPool = storagePoolMgr.getStoragePool(primaryStore.getPoolType(), primaryStore.getUuid()); - KVMPhysicalDisk disk = primaryPool.getPhysicalDisk(volume.getPath()); + KVMPhysicalDisk disk = storagePoolMgr.getPhysicalDisk(primaryStore.getPoolType(), + primaryStore.getUuid(), volume.getPath()); if (state == DomainInfo.DomainState.VIR_DOMAIN_RUNNING && !primaryPool.isExternalSnapshot()) { String vmUuid = vm.getUUIDString(); Object[] args = new Object[] { snapshotName, vmUuid };