etan39
/
cloudstack
mirror of https://github.com/apache/cloudstack.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1608 from myENA/upstream/context-cleanup 

Cleanup RBD contexts after exceptions to prevent potential agent crashWe noticed that when an exception occurs within the cleanup loop inside
the deletePhysicalDisk routine that the previously allocated contexts
are not cleaned up.  This seemed to cause an eventual crash of the host
agent after multiple exceptions within the loop.

In addition to ensuring the contexts are always freed we also improved
the logging when exceptions do occur to include the actual return code
from the underlying library in deletePhysicalDisk and deleteSnapshot.

* pr/1608:
  improve logging readability
  Cleanup rbd contexts and improve exception logging

Signed-off-by: Will Stevens <williamstevens@gmail.com>
This commit is contained in:
Will Stevens 2016-07-18 14:10:36 -04:00
parent ea48e95bdd c8fce3ff31
commit bb9d94bc64
2 changed files with 42 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java
View File

@ -69,7 +69,10 @@ import org.libvirt.LibvirtException;
import com.ceph.rados.IoCTX;
import com.ceph.rados.Rados;
import com.ceph.rados.exceptions.ErrorCode;
import com.ceph.rados.exceptions.RadosException;
import com.ceph.rbd.Rbd;
import com.ceph.rbd.RbdException;
import com.ceph.rbd.RbdImage;
import com.cloud.agent.api.Answer;
import com.cloud.agent.api.storage.PrimaryStorageDownloadAnswer;
@ -1325,6 +1328,9 @@ public class KVMStorageProcessor implements StorageProcessor {
image.snapRemove(snapshotName);
s_logger.info("Snapshot " + snap_full_name + " successfully removed from " +
primaryPool.getType().toString() + " pool.");
} catch (RbdException e) {
s_logger.error("Failed to remove snapshot " + snap_full_name + ", with exception: " + e.toString() +
", RBD error: " + ErrorCode.getErrorMessage(e.getReturnValue()));
} finally {
rbd.close(image);
r.ioCtxDestroy(io);
@ -1334,8 +1340,16 @@ public class KVMStorageProcessor implements StorageProcessor {
throw new InternalErrorException("Operation not implemented for storage pool type of " + primaryPool.getType().toString());
}
return new Answer(cmd, true, "Snapshot " + snap_full_name + " removed successfully.");
} catch (RadosException e) {
s_logger.error("Failed to remove snapshot " + snap_full_name + ", with exception: " + e.toString() +
", RBD error: " + ErrorCode.getErrorMessage(e.getReturnValue()));
return new Answer(cmd, false, "Failed to remove snapshot " + snap_full_name);
} catch (RbdException e) {
s_logger.error("Failed to remove snapshot " + snap_full_name + ", with exception: " + e.toString() +
", RBD error: " + ErrorCode.getErrorMessage(e.getReturnValue()));
return new Answer(cmd, false, "Failed to remove snapshot " + snap_full_name);
} catch (Exception e) {
s_logger.error(e.getMessage());
s_logger.error("Failed to remove snapshot " + snap_full_name + ", with exception: " + e.toString());
return new Answer(cmd, false, "Failed to remove snapshot " + snap_full_name);
}
}

43
plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java
View File

@ -35,6 +35,7 @@ import org.libvirt.StorageVol;
import com.ceph.rados.IoCTX;
import com.ceph.rados.Rados;
import com.ceph.rados.exceptions.ErrorCode;
import com.ceph.rados.exceptions.RadosException;
import com.ceph.rbd.Rbd;
import com.ceph.rbd.RbdException;
@ -863,26 +864,36 @@ public class LibvirtStorageAdaptor implements StorageAdaptor {
RbdImage image = rbd.open(uuid);
s_logger.debug("Fetching list of snapshots of RBD image " + pool.getSourceDir() + "/" + uuid);
List<RbdSnapInfo> snaps = image.snapList();
for (RbdSnapInfo snap : snaps) {
if (image.snapIsProtected(snap.name)) {
s_logger.debug("Unprotecting snapshot " + pool.getSourceDir() + "/" + uuid + "@" + snap.name);
image.snapUnprotect(snap.name);
} else {
s_logger.debug("Snapshot " + pool.getSourceDir() + "/" + uuid + "@" + snap.name + " is not protected.");
try {
for (RbdSnapInfo snap : snaps) {
if (image.snapIsProtected(snap.name)) {
s_logger.debug("Unprotecting snapshot " + pool.getSourceDir() + "/" + uuid + "@" + snap.name);
image.snapUnprotect(snap.name);
} else {
s_logger.debug("Snapshot " + pool.getSourceDir() + "/" + uuid + "@" + snap.name + " is not protected.");
}
s_logger.debug("Removing snapshot " + pool.getSourceDir() + "/" + uuid + "@" + snap.name);
image.snapRemove(snap.name);
}
s_logger.debug("Removing snapshot " + pool.getSourceDir() + "/" + uuid + "@" + snap.name);
image.snapRemove(snap.name);
s_logger.info("Succesfully unprotected and removed any remaining snapshots (" + snaps.size() + ") of "
+ pool.getSourceDir() + "/" + uuid + " Continuing to remove the RBD image");
} catch (RbdException e) {
s_logger.error("Failed to remove snapshot with exception: " + e.toString() +
", RBD error: " + ErrorCode.getErrorMessage(e.getReturnValue()));
throw new CloudRuntimeException(e.toString() + " - " + ErrorCode.getErrorMessage(e.getReturnValue()));
} finally {
s_logger.debug("Closing image and destroying context");
rbd.close(image);
r.ioCtxDestroy(io);
}
rbd.close(image);
r.ioCtxDestroy(io);
s_logger.info("Succesfully unprotected and removed any remaining snapshots (" + snaps.size() + ") of "
+ pool.getSourceDir() + "/" + uuid + " Continuing to remove the RBD image");
} catch (RadosException e) {
throw new CloudRuntimeException(e.toString());
s_logger.error("Failed to remove snapshot with exception: " + e.toString() +
", RBD error: " + ErrorCode.getErrorMessage(e.getReturnValue()));
throw new CloudRuntimeException(e.toString() + " - " + ErrorCode.getErrorMessage(e.getReturnValue()));
} catch (RbdException e) {
throw new CloudRuntimeException(e.toString());
s_logger.error("Failed to remove snapshot with exception: " + e.toString() +
", RBD error: " + ErrorCode.getErrorMessage(e.getReturnValue()));
throw new CloudRuntimeException(e.toString() + " - " + ErrorCode.getErrorMessage(e.getReturnValue()));
}
}