diff --git a/api/src/com/cloud/agent/api/storage/CreateAnswer.java b/api/src/com/cloud/agent/api/storage/CreateAnswer.java index 70c472bb2a1..647d93be15e 100644 --- a/api/src/com/cloud/agent/api/storage/CreateAnswer.java +++ b/api/src/com/cloud/agent/api/storage/CreateAnswer.java @@ -22,6 +22,7 @@ import com.cloud.agent.api.to.VolumeTO; public class CreateAnswer extends Answer { VolumeTO volume; + boolean requestTemplateReload = false; protected CreateAnswer() { } @@ -34,6 +35,11 @@ public class CreateAnswer extends Answer { super(cmd, false, details); } + public CreateAnswer(CreateCommand cmd, String details, boolean requestTemplateReload) { + super(cmd, false, details); + this.requestTemplateReload = requestTemplateReload; + } + public CreateAnswer(CreateCommand cmd, Exception e) { super(cmd, e); } @@ -41,4 +47,8 @@ public class CreateAnswer extends Answer { public VolumeTO getVolume() { return volume; } + + public boolean templateReloadRequested() { + return requestTemplateReload; + } } diff --git a/server/src/com/cloud/async/AsyncJobManagerImpl.java b/server/src/com/cloud/async/AsyncJobManagerImpl.java index abe3fc2f334..9de420b87a9 100644 --- a/server/src/com/cloud/async/AsyncJobManagerImpl.java +++ b/server/src/com/cloud/async/AsyncJobManagerImpl.java @@ -62,6 +62,7 @@ import com.cloud.utils.db.DB; import com.cloud.utils.db.GlobalLock; import com.cloud.utils.db.Transaction; import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.exception.ExceptionUtil; import com.cloud.utils.mgmt.JmxUtil; import com.cloud.utils.net.MacAddress; import com.google.gson.Gson; @@ -351,7 +352,7 @@ public class AsyncJobManagerImpl implements AsyncJobManager, ClusterManagerListe try { JmxUtil.registerMBean("AsyncJobManager", "Active Job " + job.getId(), new AsyncJobMBeanImpl(job)); } catch(Exception e) { - s_logger.warn("Unable to register active job " + job.getId() + " to JMX minotoring"); + s_logger.warn("Unable to register active job " + job.getId() + " to JMX minotoring due to exception " + ExceptionUtil.toString(e)); } BaseAsyncCmd cmdObj = null; diff --git a/server/src/com/cloud/cluster/ClusterManagerImpl.java b/server/src/com/cloud/cluster/ClusterManagerImpl.java index aa03c6135bd..14b3a03eb87 100644 --- a/server/src/com/cloud/cluster/ClusterManagerImpl.java +++ b/server/src/com/cloud/cluster/ClusterManagerImpl.java @@ -49,6 +49,7 @@ import com.cloud.utils.db.DB; import com.cloud.utils.db.Transaction; import com.cloud.utils.events.SubscriptionMgr; import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.exception.ExceptionUtil; import com.cloud.utils.mgmt.JmxUtil; import com.cloud.utils.net.MacAddress; import com.cloud.utils.net.NetUtils; @@ -618,7 +619,7 @@ public class ClusterManagerImpl implements ClusterManager { try { JmxUtil.registerMBean("ClusterManager", "Node " + mshost.getId(), new ClusterManagerMBeanImpl(this, mshost)); } catch(Exception e) { - s_logger.warn("Unable to regiester cluster node into JMX monitoring due to exception " + e.toString()); + s_logger.warn("Unable to regiester cluster node into JMX monitoring due to exception " + ExceptionUtil.toString(e)); } } } diff --git a/server/src/com/cloud/storage/StorageManagerImpl.java b/server/src/com/cloud/storage/StorageManagerImpl.java index 60adf83fc80..c723fb6126d 100755 --- a/server/src/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/com/cloud/storage/StorageManagerImpl.java @@ -119,7 +119,6 @@ import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.hypervisor.HypervisorGuruManager; import com.cloud.network.NetworkManager; import com.cloud.network.router.VirtualNetworkApplianceManager; -import com.cloud.offering.ServiceOffering; import com.cloud.server.ManagementServer; import com.cloud.service.ServiceOfferingVO; import com.cloud.service.dao.ServiceOfferingDao; @@ -179,7 +178,6 @@ import com.cloud.vm.VirtualMachine.State; import com.cloud.vm.VirtualMachine.Type; import com.cloud.vm.VirtualMachineManager; import com.cloud.vm.VirtualMachineProfile; -import com.cloud.vm.VirtualMachineProfileImpl; import com.cloud.vm.dao.ConsoleProxyDao; import com.cloud.vm.dao.DomainRouterDao; import com.cloud.vm.dao.SecondaryStorageVmDao; @@ -649,25 +647,39 @@ public class StorageManagerImpl implements StorageManager, StorageService, Manag CreateCommand cmd = null; VMTemplateStoragePoolVO tmpltStoredOn = null; - if (volume.getVolumeType() == VolumeType.ROOT && Storage.ImageFormat.ISO != template.getFormat()) { - tmpltStoredOn = _tmpltMgr.prepareTemplateForCreate(template, pool); - if (tmpltStoredOn == null) { - continue; - } - cmd = new CreateCommand(dskCh, tmpltStoredOn.getLocalDownloadPath(), new StorageFilerTO(pool)); - } else { - cmd = new CreateCommand(dskCh, new StorageFilerTO(pool)); - } - - try { - Answer answer = sendToPool(pool, cmd); - if (answer != null && answer.getResult()) { - created = ((CreateAnswer) answer).getVolume(); - break; - } - } catch (StorageUnavailableException e) { - s_logger.debug("Storage unavailable for " + pool.getId()); + + for(int i = 0; i < 2; i++) { + if (volume.getVolumeType() == VolumeType.ROOT && Storage.ImageFormat.ISO != template.getFormat()) { + tmpltStoredOn = _tmpltMgr.prepareTemplateForCreate(template, pool); + if (tmpltStoredOn == null) { + continue; + } + cmd = new CreateCommand(dskCh, tmpltStoredOn.getLocalDownloadPath(), new StorageFilerTO(pool)); + } else { + cmd = new CreateCommand(dskCh, new StorageFilerTO(pool)); + } + + try { + Answer answer = sendToPool(pool, cmd); + if (answer != null && answer.getResult()) { + created = ((CreateAnswer) answer).getVolume(); + break; + } + + if(tmpltStoredOn != null && answer != null && (answer instanceof CreateAnswer) && ((CreateAnswer)answer).templateReloadRequested()) { + if(!_tmpltMgr.resetTemplateDownloadStateOnPool(tmpltStoredOn.getId())) + break; // break out of template-redeploy retry loop + } else { + break; + } + } catch (StorageUnavailableException e) { + s_logger.debug("Storage unavailable for " + pool.getId()); + break; // break out of template-redeploy retry loop + } } + + if(created != null) + break; s_logger.debug("Retrying the create because it failed on pool " + pool); } @@ -694,8 +706,6 @@ public class StorageManagerImpl implements StorageManager, StorageService, Manag volume.setPodId(pod.getId()); volume.setState(Volume.State.Ready); _volsDao.persist(volume); - - } txn.commit(); return volume; @@ -2544,22 +2554,32 @@ public class StorageManagerImpl implements StorageManager, StorageService, Manag } CreateCommand cmd = null; - if (template != null && template.getFormat() != Storage.ImageFormat.ISO) { - VMTemplateStoragePoolVO tmpltStoredOn = null; - tmpltStoredOn = _tmpltMgr.prepareTemplateForCreate(template, pool); - if (tmpltStoredOn == null) { - s_logger.debug("Cannot use this pool " + pool + " because we can't propagate template " + template); - return null; - } - cmd = new CreateCommand(diskProfile, tmpltStoredOn.getLocalDownloadPath(), new StorageFilerTO(pool)); - } else { - cmd = new CreateCommand(diskProfile, new StorageFilerTO(pool)); - } - long[] hostIdsToTryFirst = {dest.getHost().getId()}; - Answer answer = sendToPool(pool, hostIdsToTryFirst, cmd); - if (answer.getResult()) { - CreateAnswer createAnswer = (CreateAnswer) answer; - return new Pair(createAnswer.getVolume(), pool); + VMTemplateStoragePoolVO tmpltStoredOn = null; + + for(int i = 0; i < 2; i++) { + if (template != null && template.getFormat() != Storage.ImageFormat.ISO) { + tmpltStoredOn = _tmpltMgr.prepareTemplateForCreate(template, pool); + if (tmpltStoredOn == null) { + s_logger.debug("Cannot use this pool " + pool + " because we can't propagate template " + template); + return null; + } + cmd = new CreateCommand(diskProfile, tmpltStoredOn.getLocalDownloadPath(), new StorageFilerTO(pool)); + } else { + cmd = new CreateCommand(diskProfile, new StorageFilerTO(pool)); + } + long[] hostIdsToTryFirst = {dest.getHost().getId()}; + Answer answer = sendToPool(pool, hostIdsToTryFirst, cmd); + if (answer.getResult()) { + CreateAnswer createAnswer = (CreateAnswer) answer; + return new Pair(createAnswer.getVolume(), pool); + } else { + if(tmpltStoredOn != null && (answer instanceof CreateAnswer) && ((CreateAnswer)answer).templateReloadRequested()) { + if(!_tmpltMgr.resetTemplateDownloadStateOnPool(tmpltStoredOn.getId())) + break; // break out of template-redeploy retry loop + } else { + break; + } + } } } diff --git a/server/src/com/cloud/template/TemplateManager.java b/server/src/com/cloud/template/TemplateManager.java index 8a5971847cf..27fd46e4e1e 100755 --- a/server/src/com/cloud/template/TemplateManager.java +++ b/server/src/com/cloud/template/TemplateManager.java @@ -70,6 +70,8 @@ public interface TemplateManager { */ VMTemplateStoragePoolVO prepareTemplateForCreate(VMTemplateVO template, StoragePool pool); + boolean resetTemplateDownloadStateOnPool(long templateStoragePoolRefId); + /** * Copies a template from its current secondary storage server to the secondary storage server in the specified zone. * @param templateId diff --git a/server/src/com/cloud/template/TemplateManagerImpl.java b/server/src/com/cloud/template/TemplateManagerImpl.java index 5cde61eb696..94676db53a6 100755 --- a/server/src/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/com/cloud/template/TemplateManagerImpl.java @@ -714,6 +714,27 @@ public class TemplateManagerImpl implements TemplateManager, Manager, TemplateSe return null; } + @Override + @DB + public boolean resetTemplateDownloadStateOnPool(long templateStoragePoolRefId) { + // have to use the same lock that prepareTemplateForCreate use to maintain state consistency + VMTemplateStoragePoolVO templateStoragePoolRef = _tmpltPoolDao.acquireInLockTable(templateStoragePoolRefId, 1200); + + if (templateStoragePoolRef == null) { + s_logger.warn("resetTemplateDownloadStateOnPool failed - unable to lock TemplateStorgePoolRef " + templateStoragePoolRefId); + return false; + } + + try { + templateStoragePoolRef.setDownloadState(VMTemplateStorageResourceAssoc.Status.NOT_DOWNLOADED); + _tmpltPoolDao.update(templateStoragePoolRefId, templateStoragePoolRef); + } finally { + _tmpltPoolDao.releaseFromLockTable(templateStoragePoolRefId); + } + + return true; + } + @Override @DB public boolean copy(long userId, long templateId, long sourceZoneId, long destZoneId) throws StorageUnavailableException, ResourceAllocationException {