CLOUDSTACK-1846, CLOUDSTACK-1845 - KVM Storage, sometimes KVMHA will remount

deleted NFS pools, causing failures when defining new storage pools. Sometimes
a storage pool has never been used on a host, and getStoragePool fails when
copying templates or in storage migration. deleteStoragePool(pool) often fails
silently, leaving no pool defined in libvirt, but a mountpoint left behind.
This patch handles some of these exceptions and brings forward any issues via
logging.

Signed-off-by: Marcus Sorensen <marcus@betterservers.com> 1364603486 -0600

plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java

@@ -1218,10 +1218,23 @@ ServerResource {
         StorageFilerTO pool = cmd.getPool();
         String secondaryStorageUrl = cmd.getSecondaryStorageURL();
         KVMStoragePool secondaryStoragePool = null;
+        KVMStoragePool primaryPool = null;
         try {
-            KVMStoragePool primaryPool = _storagePoolMgr.getStoragePool(
-                    pool.getType(),
-                    pool.getUuid());
+            try {
+                primaryPool = _storagePoolMgr.getStoragePool(
+                     pool.getType(),
+                     pool.getUuid());
+            } catch (CloudRuntimeException e) {
+                if (e.getMessage().contains("not found")) {
+                    primaryPool = _storagePoolMgr.createStoragePool(cmd.getPool().getUuid(),
+                                      cmd.getPool().getHost(), cmd.getPool().getPort(),
+                                      cmd.getPool().getPath(), cmd.getPool().getUserInfo(),
+                                      cmd.getPool().getType());
+                } else {
+                    return new CopyVolumeAnswer(cmd, false, e.getMessage(), null, null);
+                }
+            }
+
             String volumeName = UUID.randomUUID().toString();
 
             if (copyToSecondary) {
@@ -2155,6 +2168,7 @@ ServerResource {
         String secondaryStorageURL = cmd.getSecondaryStorageUrl();
 
         KVMStoragePool secondaryStorage = null;
+        KVMStoragePool primary = null;
         try {
             Connect conn = LibvirtConnection.getConnection();
             String templateFolder = cmd.getAccountId() + File.separator
@@ -2164,9 +2178,21 @@ ServerResource {
             secondaryStorage = _storagePoolMgr.getStoragePoolByURI(
                     secondaryStorageURL);
 
-            KVMStoragePool primary = _storagePoolMgr.getStoragePool(
+            try {
+                primary = _storagePoolMgr.getStoragePool(
                     cmd.getPool().getType(),
                     cmd.getPrimaryStoragePoolNameLabel());
+            } catch (CloudRuntimeException e) {
+                if (e.getMessage().contains("not found")) {
+                    primary = _storagePoolMgr.createStoragePool(cmd.getPool().getUuid(),
+                                      cmd.getPool().getHost(), cmd.getPool().getPort(),
+                                      cmd.getPool().getPath(), cmd.getPool().getUserInfo(),
+                                      cmd.getPool().getType());
+                } else {
+                    return new CreatePrivateTemplateAnswer(cmd, false, e.getMessage());
+                }
+            }
+
             KVMPhysicalDisk disk = primary.getPhysicalDisk(cmd.getVolumePath());
             String tmpltPath = secondaryStorage.getLocalPath() + File.separator
                     + templateInstallFolder;

plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java

@@ -124,6 +124,23 @@ public class LibvirtStorageAdaptor implements StorageAdaptor {
             return sp;
         } catch (LibvirtException e) {
             s_logger.error(e.toString());
+            // if error is that pool is mounted, try to handle it
+            if (e.toString().contains("already mounted")) {
+                s_logger.error("Attempting to unmount old mount libvirt is unaware of at "+targetPath);
+                String result = Script.runSimpleBashScript("umount " + targetPath );
+                if (result == null) {
+                    s_logger.error("Succeeded in unmounting " + targetPath);
+                    try {
+                        sp = conn.storagePoolCreateXML(spd.toString(), 0);
+                        s_logger.error("Succeeded in redefining storage");
+                        return sp;
+                    } catch (LibvirtException l) {
+                        s_logger.error("Target was already mounted, unmounted it but failed to redefine storage:" + l);
+                    }
+                } else {
+                    s_logger.error("Failed in unmounting and redefining storage");
+                }
+            }
             if (sp != null) {
                 try {
                     if (sp.isPersistent() == 1) {
@@ -134,8 +151,8 @@ public class LibvirtStorageAdaptor implements StorageAdaptor {
                     }
                     sp.free();
                 } catch (LibvirtException l) {
-                    s_logger.debug("Failed to define nfs storage pool with: "
-                            + l.toString());
+                    s_logger.debug("Failed to undefine nfs storage pool with: "
+                        + l.toString());
                 }
             }
             return null;
@@ -541,6 +558,19 @@ public class LibvirtStorageAdaptor implements StorageAdaptor {
             }
             return true;
         } catch (LibvirtException e) {
+            // handle ebusy error when pool is quickly destroyed
+            if (e.toString().contains("exit status 16")) {
+                String targetPath = _mountPoint + File.separator + uuid;
+                s_logger.error("deleteStoragePool removed pool from libvirt, but libvirt had trouble"
+                               + "unmounting the pool. Trying umount location " + targetPath
+                               + "again in a few seconds");
+                String result = Script.runSimpleBashScript("sleep 5 && umount " + targetPath );
+                if (result == null) {
+                    s_logger.error("Succeeded in unmounting " + targetPath);
+                    return true;
+                }
+                s_logger.error("failed in umount retry");
+            }
             throw new CloudRuntimeException(e.toString());
         }
     }
@@ -766,17 +796,7 @@ public class LibvirtStorageAdaptor implements StorageAdaptor {
 
     @Override
     public boolean deleteStoragePool(KVMStoragePool pool) {
-        LibvirtStoragePool libvirtPool = (LibvirtStoragePool) pool;
-        StoragePool virtPool = libvirtPool.getPool();
-        try {
-            virtPool.destroy();
-            virtPool.undefine();
-            virtPool.free();
-        } catch (LibvirtException e) {
-            return false;
-        }
-
-        return true;
+        return deleteStoragePool(pool.getUuid());
     }
 
     public boolean deleteVbdByPath(String diskPath) {