From 3e857bde08241a6e45ba98dcde8885242665d676 Mon Sep 17 00:00:00 2001 From: Koushik Das Date: Wed, 21 Jan 2015 16:02:32 +0530 Subject: [PATCH] volume-upload: management server polling and upload status from agent Fixed some issues with the polling logic to handle scenarios when SSVM is destroyed/down. Also changed the status of the volume_store_ref entries so that subsequent operations work fine. --- .../agent/manager/MockStorageManagerImpl.java | 2 +- .../storage/ImageStoreUploadMonitorImpl.java | 194 ++++++++++-------- 2 files changed, 114 insertions(+), 82 deletions(-) diff --git a/plugins/hypervisors/simulator/src/com/cloud/agent/manager/MockStorageManagerImpl.java b/plugins/hypervisors/simulator/src/com/cloud/agent/manager/MockStorageManagerImpl.java index 306826d6109..ba24dc7310c 100644 --- a/plugins/hypervisors/simulator/src/com/cloud/agent/manager/MockStorageManagerImpl.java +++ b/plugins/hypervisors/simulator/src/com/cloud/agent/manager/MockStorageManagerImpl.java @@ -1260,6 +1260,6 @@ public class MockStorageManagerImpl extends ManagerBase implements MockStorageMa @Override public UploadStatusAnswer getUploadStatus(UploadStatusCommand cmd) { - return new UploadStatusAnswer(cmd, UploadStatus.UNKNOWN); + return new UploadStatusAnswer(cmd, UploadStatus.COMPLETED); } } diff --git a/server/src/com/cloud/storage/ImageStoreUploadMonitorImpl.java b/server/src/com/cloud/storage/ImageStoreUploadMonitorImpl.java index 63642f0db93..e0af472258b 100755 --- a/server/src/com/cloud/storage/ImageStoreUploadMonitorImpl.java +++ b/server/src/com/cloud/storage/ImageStoreUploadMonitorImpl.java @@ -32,6 +32,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector; +import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine.State; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; import org.apache.cloudstack.managed.context.ManagedContextRunnable; @@ -187,14 +188,18 @@ public class ImageStoreUploadMonitorImpl extends ManagerBase implements ImageSto continue; } Host host = _hostDao.findById(ep.getId()); - if (host != null && host.getManagementServerId() != null && _nodeId == host.getManagementServerId().longValue()) { - UploadStatusCommand cmd = new UploadStatusCommand(volume.getId(), EntityType.Volume); - Answer answer = ep.sendMessage(cmd); - if (answer == null || !(answer instanceof UploadStatusAnswer)) { - s_logger.warn("No or invalid answer corresponding to UploadStatusCommand for volume " + volumeDataStore.getVolumeId()); - continue; + if (host != null && host.getManagementServerId() != null) { + if (_nodeId == host.getManagementServerId().longValue()) { + UploadStatusCommand cmd = new UploadStatusCommand(volume.getId(), EntityType.Volume); + Answer answer = ep.sendMessage(cmd); + if (answer == null || !(answer instanceof UploadStatusAnswer)) { + s_logger.warn("No or invalid answer corresponding to UploadStatusCommand for volume " + volumeDataStore.getVolumeId()); + continue; + } + handleVolumeStatusResponse((UploadStatusAnswer)answer, volume, volumeDataStore); } - handleVolumeStatusResponse((UploadStatusAnswer)answer, volume, volumeDataStore); + } else { + handleVolumeStatusResponse(null, volume, volumeDataStore); } } @@ -213,14 +218,18 @@ public class ImageStoreUploadMonitorImpl extends ManagerBase implements ImageSto continue; } Host host = _hostDao.findById(ep.getId()); - if (host != null && host.getManagementServerId() != null && _nodeId == host.getManagementServerId().longValue()) { - UploadStatusCommand cmd = new UploadStatusCommand(template.getId(), EntityType.Template); - Answer answer = ep.sendMessage(cmd); - if (answer == null || !(answer instanceof UploadStatusAnswer)) { - s_logger.warn("No or invalid answer corresponding to UploadStatusCommand for template " + templateDataStore.getTemplateId()); - continue; + if (host != null && host.getManagementServerId() != null) { + if (_nodeId == host.getManagementServerId().longValue()) { + UploadStatusCommand cmd = new UploadStatusCommand(template.getId(), EntityType.Template); + Answer answer = ep.sendMessage(cmd); + if (answer == null || !(answer instanceof UploadStatusAnswer)) { + s_logger.warn("No or invalid answer corresponding to UploadStatusCommand for template " + templateDataStore.getTemplateId()); + continue; + } + handleTemplateStatusResponse((UploadStatusAnswer)answer, template, templateDataStore); } - handleTemplateStatusResponse((UploadStatusAnswer)answer, template, templateDataStore); + } else { + handleTemplateStatusResponse(null, template, templateDataStore); } } } @@ -233,47 +242,58 @@ public class ImageStoreUploadMonitorImpl extends ManagerBase implements ImageSto VolumeVO tmpVolume = _volumeDao.findById(volume.getId()); VolumeDataStoreVO tmpVolumeDataStore = _volumeDataStoreDao.findById(volumeDataStore.getId()); try { - switch (answer.getStatus()) { - case COMPLETED: - tmpVolumeDataStore.setDownloadState(VMTemplateStorageResourceAssoc.Status.UPLOADED); - stateMachine.transitTo(tmpVolume, Event.OperationSucceeded, null, _volumeDao); - if (s_logger.isDebugEnabled()) { - s_logger.debug("Volume " + tmpVolume.getUuid() + " uploaded successfully"); - } - break; - case IN_PROGRESS: - if (tmpVolume.getState() == Volume.State.NotUploaded) { - tmpVolumeDataStore.setDownloadState(VMTemplateStorageResourceAssoc.Status.UPLOAD_IN_PROGRESS); - stateMachine.transitTo(tmpVolume, Event.UploadRequested, null, _volumeDao); - } else if (tmpVolume.getState() == Volume.State.UploadInProgress) { // check for timeout - if (System.currentTimeMillis() - tmpVolumeDataStore.getCreated().getTime() > _uploadOperationTimeout) { - tmpVolumeDataStore.setDownloadState(VMTemplateStorageResourceAssoc.Status.UPLOAD_ERROR); - stateMachine.transitTo(tmpVolume, Event.OperationFailed, null, _volumeDao); - if (s_logger.isDebugEnabled()) { - s_logger.debug("Volume " + tmpVolume.getUuid() + " failed to upload due to operation timed out"); + if (answer != null) { + switch (answer.getStatus()) { + case COMPLETED: + tmpVolumeDataStore.setDownloadState(VMTemplateStorageResourceAssoc.Status.DOWNLOADED); + tmpVolumeDataStore.setState(State.Ready); + stateMachine.transitTo(tmpVolume, Event.OperationSucceeded, null, _volumeDao); + if (s_logger.isDebugEnabled()) { + s_logger.debug("Volume " + tmpVolume.getUuid() + " uploaded successfully"); + } + break; + case IN_PROGRESS: + if (tmpVolume.getState() == Volume.State.NotUploaded) { + tmpVolumeDataStore.setDownloadState(VMTemplateStorageResourceAssoc.Status.DOWNLOAD_IN_PROGRESS); + stateMachine.transitTo(tmpVolume, Event.UploadRequested, null, _volumeDao); + } else if (tmpVolume.getState() == Volume.State.UploadInProgress) { // check for timeout + if (System.currentTimeMillis() - tmpVolumeDataStore.getCreated().getTime() > _uploadOperationTimeout) { + tmpVolumeDataStore.setDownloadState(VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR); + stateMachine.transitTo(tmpVolume, Event.OperationFailed, null, _volumeDao); + if (s_logger.isDebugEnabled()) { + s_logger.debug("Volume " + tmpVolume.getUuid() + " failed to upload due to operation timed out"); + } } } + break; + case ERROR: + tmpVolumeDataStore.setDownloadState(VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR); + tmpVolumeDataStore.setState(State.Failed); + stateMachine.transitTo(tmpVolume, Event.OperationFailed, null, _volumeDao); + if (s_logger.isDebugEnabled()) { + s_logger.debug("Volume " + tmpVolume.getUuid() + " failed to upload. Error details: " + answer.getDetails()); + } + break; + case UNKNOWN: + if (tmpVolume.getState() == Volume.State.NotUploaded) { // check for timeout + if (System.currentTimeMillis() - tmpVolumeDataStore.getCreated().getTime() > _uploadOperationTimeout) { + tmpVolumeDataStore.setDownloadState(VMTemplateStorageResourceAssoc.Status.ABANDONED); + tmpVolumeDataStore.setState(State.Failed); + stateMachine.transitTo(tmpVolume, Event.OperationTimeout, null, _volumeDao); + if (s_logger.isDebugEnabled()) { + s_logger.debug("Volume " + tmpVolume.getUuid() + " failed to upload due to operation timed out"); + } + } + } + break; } - break; - case ERROR: + } else { tmpVolumeDataStore.setDownloadState(VMTemplateStorageResourceAssoc.Status.UPLOAD_ERROR); + tmpVolumeDataStore.setState(State.Failed); stateMachine.transitTo(tmpVolume, Event.OperationFailed, null, _volumeDao); if (s_logger.isDebugEnabled()) { - s_logger.debug("Volume " + tmpVolume.getUuid() + " failed to upload. Error details: " + answer.getDetails()); + s_logger.debug("Volume " + tmpVolume.getUuid() + " failed to upload as SSVM is either destroyed or SSVM agent not in 'Up' state"); } - break; - case UNKNOWN: - // check for timeout - if (tmpVolume.getState() == Volume.State.NotUploaded) { - if (System.currentTimeMillis() - tmpVolumeDataStore.getCreated().getTime() > _uploadOperationTimeout) { - tmpVolumeDataStore.setDownloadState(VMTemplateStorageResourceAssoc.Status.ABANDONED); - stateMachine.transitTo(tmpVolume, Event.OperationTimeout, null, _volumeDao); - if (s_logger.isDebugEnabled()) { - s_logger.debug("Volume " + tmpVolume.getUuid() + " failed to upload due to operation timed out"); - } - } - } - break; } _volumeDataStoreDao.update(tmpVolumeDataStore.getId(), tmpVolumeDataStore); } catch (NoTransitionException e) { @@ -291,46 +311,58 @@ public class ImageStoreUploadMonitorImpl extends ManagerBase implements ImageSto VMTemplateVO tmpTemplate = _templateDao.findById(template.getId()); TemplateDataStoreVO tmpTemplateDataStore = _templateDataStoreDao.findById(templateDataStore.getId()); try { - switch (answer.getStatus()) { - case COMPLETED: - tmpTemplateDataStore.setDownloadState(VMTemplateStorageResourceAssoc.Status.UPLOADED); - stateMachine.transitTo(tmpTemplate, VirtualMachineTemplate.Event.OperationSucceeded, null, _templateDao); - if (s_logger.isDebugEnabled()) { - s_logger.debug("Template " + tmpTemplate.getUuid() + " uploaded successfully"); - } - break; - case IN_PROGRESS: - if (tmpTemplate.getState() == VirtualMachineTemplate.State.NotUploaded) { - tmpTemplateDataStore.setDownloadState(VMTemplateStorageResourceAssoc.Status.UPLOAD_IN_PROGRESS); - stateMachine.transitTo(tmpTemplate, VirtualMachineTemplate.Event.UploadRequested, null, _templateDao); - } else if (tmpTemplate.getState() == VirtualMachineTemplate.State.UploadInProgress) { // check for timeout - if (System.currentTimeMillis() - tmpTemplateDataStore.getCreated().getTime() > _uploadOperationTimeout) { - tmpTemplateDataStore.setDownloadState(VMTemplateStorageResourceAssoc.Status.UPLOAD_ERROR); - stateMachine.transitTo(tmpTemplate, VirtualMachineTemplate.Event.OperationFailed, null, _templateDao); - if (s_logger.isDebugEnabled()) { - s_logger.debug("Template " + tmpTemplate.getUuid() + " failed to upload due to operation timed out"); + if (answer != null) { + switch (answer.getStatus()) { + case COMPLETED: + tmpTemplateDataStore.setDownloadState(VMTemplateStorageResourceAssoc.Status.DOWNLOADED); + tmpTemplateDataStore.setState(State.Ready); + stateMachine.transitTo(tmpTemplate, VirtualMachineTemplate.Event.OperationSucceeded, null, _templateDao); + if (s_logger.isDebugEnabled()) { + s_logger.debug("Template " + tmpTemplate.getUuid() + " uploaded successfully"); + } + break; + case IN_PROGRESS: + if (tmpTemplate.getState() == VirtualMachineTemplate.State.NotUploaded) { + tmpTemplateDataStore.setDownloadState(VMTemplateStorageResourceAssoc.Status.DOWNLOAD_IN_PROGRESS); + stateMachine.transitTo(tmpTemplate, VirtualMachineTemplate.Event.UploadRequested, null, _templateDao); + } else if (tmpTemplate.getState() == VirtualMachineTemplate.State.UploadInProgress) { // check for timeout + if (System.currentTimeMillis() - tmpTemplateDataStore.getCreated().getTime() > _uploadOperationTimeout) { + tmpTemplateDataStore.setDownloadState(VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR); + stateMachine.transitTo(tmpTemplate, VirtualMachineTemplate.Event.OperationFailed, null, _templateDao); + if (s_logger.isDebugEnabled()) { + s_logger.debug("Template " + tmpTemplate.getUuid() + " failed to upload due to operation timed out"); + } } } + break; + case ERROR: + tmpTemplateDataStore.setDownloadState(VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR); + tmpTemplateDataStore.setState(State.Failed); + stateMachine.transitTo(tmpTemplate, VirtualMachineTemplate.Event.OperationFailed, null, _templateDao); + if (s_logger.isDebugEnabled()) { + s_logger.debug("Template " + tmpTemplate.getUuid() + " failed to upload. Error details: " + answer.getDetails()); + } + break; + case UNKNOWN: + if (tmpTemplate.getState() == VirtualMachineTemplate.State.NotUploaded) { // check for timeout + if (System.currentTimeMillis() - tmpTemplateDataStore.getCreated().getTime() > _uploadOperationTimeout) { + tmpTemplateDataStore.setDownloadState(VMTemplateStorageResourceAssoc.Status.ABANDONED); + tmpTemplateDataStore.setState(State.Failed); + stateMachine.transitTo(tmpTemplate, VirtualMachineTemplate.Event.OperationTimeout, null, _templateDao); + if (s_logger.isDebugEnabled()) { + s_logger.debug("Template " + tmpTemplate.getUuid() + " failed to upload due to operation timed out"); + } + } + } + break; } - break; - case ERROR: + } else { tmpTemplateDataStore.setDownloadState(VMTemplateStorageResourceAssoc.Status.UPLOAD_ERROR); + tmpTemplateDataStore.setState(State.Failed); stateMachine.transitTo(tmpTemplate, VirtualMachineTemplate.Event.OperationFailed, null, _templateDao); if (s_logger.isDebugEnabled()) { - s_logger.debug("Template " + tmpTemplate.getUuid() + " failed to upload. Error details: " + answer.getDetails()); + s_logger.debug("Template " + tmpTemplate.getUuid() + " failed to upload as SSVM is either destroyed or SSVM agent not in 'Up' state"); } - break; - case UNKNOWN: - if (tmpTemplate.getState() == VirtualMachineTemplate.State.NotUploaded) { // check for timeout - if (System.currentTimeMillis() - tmpTemplateDataStore.getCreated().getTime() > _uploadOperationTimeout) { - tmpTemplateDataStore.setDownloadState(VMTemplateStorageResourceAssoc.Status.ABANDONED); - stateMachine.transitTo(tmpTemplate, VirtualMachineTemplate.Event.OperationTimeout, null, _templateDao); - if (s_logger.isDebugEnabled()) { - s_logger.debug("Template " + tmpTemplate.getUuid() + " failed to upload due to operation timed out"); - } - } - } - break; } _templateDataStoreDao.update(tmpTemplateDataStore.getId(), tmpTemplateDataStore); } catch (NoTransitionException e) {