From 99be3718cfa6af0899a95ea4cc7eadfdbafcddf4 Mon Sep 17 00:00:00 2001 From: anthony Date: Thu, 19 May 2011 19:06:07 -0700 Subject: [PATCH] bug 3224: multiple secondary storage, copying template between zone works --- .../agent/api/storage/DownloadCommand.java | 10 +- .../com/cloud/api/commands/CopyIsoCmd.java | 125 ----------- .../com/cloud/template/TemplateService.java | 3 - client/tomcatconf/commands.properties.in | 2 +- .../com/cloud/storage/JavaStorageLayer.java | 20 +- core/src/com/cloud/storage/StorageLayer.java | 2 + .../resource/NfsSecondaryStorageResource.java | 20 +- .../storage/template/DownloadManagerImpl.java | 3 + .../src/com/cloud/storage/StorageManager.java | 4 +- .../com/cloud/storage/StorageManagerImpl.java | 15 ++ .../storage/download/DownloadMonitor.java | 2 +- .../storage/download/DownloadMonitorImpl.java | 83 +++++--- .../SecondaryStorageManagerImpl.java | 4 + .../com/cloud/template/TemplateManager.java | 27 +-- .../cloud/template/TemplateManagerImpl.java | 199 ++++++++---------- 15 files changed, 217 insertions(+), 302 deletions(-) delete mode 100755 api/src/com/cloud/api/commands/CopyIsoCmd.java diff --git a/api/src/com/cloud/agent/api/storage/DownloadCommand.java b/api/src/com/cloud/agent/api/storage/DownloadCommand.java index 17a1f87887b..a0dfb9ee9cd 100644 --- a/api/src/com/cloud/agent/api/storage/DownloadCommand.java +++ b/api/src/com/cloud/agent/api/storage/DownloadCommand.java @@ -61,6 +61,7 @@ public class DownloadCommand extends AbstractDownloadCommand { this.id = that.id; this.description = that.description; this.auth = that.getAuth(); + this.setSecUrl(that.getSecUrl()); this.maxDownloadSizeInBytes = that.getMaxDownloadSizeInBytes(); } @@ -75,8 +76,13 @@ public class DownloadCommand extends AbstractDownloadCommand { } public DownloadCommand(String secUrl, String url, VirtualMachineTemplate template, String user, String passwd, Long maxDownloadSizeInBytes) { - this(secUrl, template, maxDownloadSizeInBytes); - this.setUrl(url); + super(template.getUniqueName(), url, template.getFormat(), template.getAccountId()); + this.hvm = template.isRequiresHvm(); + this.checksum = template.getChecksum(); + this.id = template.getId(); + this.description = template.getDisplayText(); + this.setSecUrl(secUrl); + this.maxDownloadSizeInBytes = maxDownloadSizeInBytes; auth = new PasswordAuth(user, passwd); } diff --git a/api/src/com/cloud/api/commands/CopyIsoCmd.java b/api/src/com/cloud/api/commands/CopyIsoCmd.java deleted file mode 100755 index a1037b40e85..00000000000 --- a/api/src/com/cloud/api/commands/CopyIsoCmd.java +++ /dev/null @@ -1,125 +0,0 @@ -/** - * Copyright (C) 2010 Cloud.com, Inc. All rights reserved. - * - * This software is licensed under the GNU General Public License v3 or later. - * - * It is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or any later version. - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - * - */ - -package com.cloud.api.commands; - -import org.apache.log4j.Logger; - -import com.cloud.api.ApiConstants; -import com.cloud.api.BaseAsyncCmd; -import com.cloud.api.BaseCmd; -import com.cloud.api.Implementation; -import com.cloud.api.Parameter; -import com.cloud.api.ServerApiException; -import com.cloud.api.response.TemplateResponse; -import com.cloud.async.AsyncJob; -import com.cloud.event.EventTypes; -import com.cloud.exception.ResourceAllocationException; -import com.cloud.exception.StorageUnavailableException; -import com.cloud.template.VirtualMachineTemplate; -import com.cloud.user.Account; - -@Implementation(description="Copies an ISO file.", responseObject=TemplateResponse.class) -public class CopyIsoCmd extends BaseAsyncCmd { - public static final Logger s_logger = Logger.getLogger(CopyIsoCmd.class.getName()); - private static final String s_name = "copyisoresponse"; - - ///////////////////////////////////////////////////// - //////////////// API parameters ///////////////////// - ///////////////////////////////////////////////////// - - @Parameter(name="destzoneid", type=CommandType.LONG, required=true, description="the ID of the destination zone to which the ISO file will be copied") - private Long destZoneId; - - @Parameter(name=ApiConstants.ID, type=CommandType.LONG, required=true, description="the ID of the ISO file") - private Long id; - - @Parameter(name="sourcezoneid", type=CommandType.LONG, required=true, description="the ID of the source zone from which the ISO file will be copied") - private Long sourceZoneId; - - ///////////////////////////////////////////////////// - /////////////////// Accessors /////////////////////// - ///////////////////////////////////////////////////// - - public Long getDestinationZoneId() { - return destZoneId; - } - - public Long getId() { - return id; - } - - public Long getSourceZoneId() { - return sourceZoneId; - } - - ///////////////////////////////////////////////////// - /////////////// API Implementation/////////////////// - ///////////////////////////////////////////////////// - - @Override - public String getCommandName() { - return s_name; - } - - public static String getStaticName() { - return s_name; - } - - @Override - public long getEntityOwnerId() { - VirtualMachineTemplate iso = _entityMgr.findById(VirtualMachineTemplate.class, getId()); - if (iso != null) { - return iso.getAccountId(); - } - - // bad id given, parent this command to SYSTEM so ERROR events are tracked - return Account.ACCOUNT_ID_SYSTEM; - } - - @Override - public String getEventType() { - return EventTypes.EVENT_ISO_COPY; - } - - @Override - public String getEventDescription() { - return "copying ISO: " + getId() + " from zone: " + getSourceZoneId() + " to zone: " + getDestinationZoneId(); - } - - public AsyncJob.Type getInstanceType() { - return AsyncJob.Type.Iso; - } - - public Long getInstanceId() { - return getId(); - } - - @Override - public void execute() throws ResourceAllocationException{ - try { - VirtualMachineTemplate iso = _templateService.copyIso(this); - TemplateResponse isoResponse = _responseGenerator.createIsoResponse3(iso, destZoneId); - isoResponse.setResponseName(getCommandName()); - this.setResponseObject(isoResponse); - } catch (StorageUnavailableException ex) { - s_logger.warn("Exception: ", ex); - throw new ServerApiException(BaseCmd.RESOURCE_UNAVAILABLE_ERROR, ex.getMessage()); - } - } -} diff --git a/api/src/com/cloud/template/TemplateService.java b/api/src/com/cloud/template/TemplateService.java index c733ecc0f3a..19bb1de9401 100755 --- a/api/src/com/cloud/template/TemplateService.java +++ b/api/src/com/cloud/template/TemplateService.java @@ -20,7 +20,6 @@ package com.cloud.template; import java.net.URISyntaxException; import com.cloud.api.commands.AttachIsoCmd; -import com.cloud.api.commands.CopyIsoCmd; import com.cloud.api.commands.CopyTemplateCmd; import com.cloud.api.commands.DeleteIsoCmd; import com.cloud.api.commands.DeleteTemplateCmd; @@ -39,8 +38,6 @@ public interface TemplateService { VirtualMachineTemplate registerIso(RegisterIsoCmd cmd) throws IllegalArgumentException, ResourceAllocationException; - VirtualMachineTemplate copyIso(CopyIsoCmd cmd) throws StorageUnavailableException, ResourceAllocationException; - VirtualMachineTemplate copyTemplate(CopyTemplateCmd cmd) throws StorageUnavailableException, ResourceAllocationException; boolean detachIso(DetachIsoCmd cmd); diff --git a/client/tomcatconf/commands.properties.in b/client/tomcatconf/commands.properties.in index a4c8ed9ea85..291d8f13df3 100755 --- a/client/tomcatconf/commands.properties.in +++ b/client/tomcatconf/commands.properties.in @@ -74,7 +74,7 @@ listIsos=com.cloud.api.commands.ListIsosCmd;15 registerIso=com.cloud.api.commands.RegisterIsoCmd;15 updateIso=com.cloud.api.commands.UpdateIsoCmd;15 deleteIso=com.cloud.api.commands.DeleteIsoCmd;15 -copyIso=com.cloud.api.commands.CopyIsoCmd;15 +copyIso=com.cloud.api.commands.CopyTemplateCmd;15 updateIsoPermissions=com.cloud.api.commands.UpdateIsoPermissionsCmd;15 listIsoPermissions=com.cloud.api.commands.ListIsoPermissionsCmd;15 extractIso=com.cloud.api.commands.ExtractIsoCmd;15 diff --git a/core/src/com/cloud/storage/JavaStorageLayer.java b/core/src/com/cloud/storage/JavaStorageLayer.java index 47cf6cd2caa..44d417c55ca 100644 --- a/core/src/com/cloud/storage/JavaStorageLayer.java +++ b/core/src/com/cloud/storage/JavaStorageLayer.java @@ -89,7 +89,25 @@ public class JavaStorageLayer implements StorageLayer { return file.delete(); } } - + + @Override + public boolean deleteDir(String dir) { + File Dir = new File(dir); + if ( !Dir.isDirectory() ) { + return false; + } + + synchronized(dir.intern()) { + File[] files = Dir.listFiles(); + for( File file : files) { + if(!file.delete() ) { + return false; + } + } + } + return true; + } + @Override public boolean exists(String path) { synchronized(path.intern()) { diff --git a/core/src/com/cloud/storage/StorageLayer.java b/core/src/com/cloud/storage/StorageLayer.java index 532a9bcf6b9..41a0cb3f3c1 100644 --- a/core/src/com/cloud/storage/StorageLayer.java +++ b/core/src/com/cloud/storage/StorageLayer.java @@ -147,4 +147,6 @@ public interface StorageLayer extends Manager { * @return true if the file was set to be both world readable and writeable */ boolean setWorldReadableAndWriteable(File file); + + boolean deleteDir(String dir); } diff --git a/core/src/com/cloud/storage/resource/NfsSecondaryStorageResource.java b/core/src/com/cloud/storage/resource/NfsSecondaryStorageResource.java index cbdfaa7ea0d..605cd027d23 100755 --- a/core/src/com/cloud/storage/resource/NfsSecondaryStorageResource.java +++ b/core/src/com/cloud/storage/resource/NfsSecondaryStorageResource.java @@ -30,7 +30,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Random; + import java.util.UUID; import javax.naming.ConfigurationException; @@ -49,6 +49,7 @@ import com.cloud.agent.api.PingStorageCommand; import com.cloud.agent.api.ReadyAnswer; import com.cloud.agent.api.ReadyCommand; import com.cloud.agent.api.SecStorageFirewallCfgCommand; +import com.cloud.agent.api.SecStorageSetupAnswer; import com.cloud.agent.api.SecStorageSetupCommand; import com.cloud.agent.api.StartupSecondaryStorageCommand; import com.cloud.agent.api.SecStorageFirewallCfgCommand.PortConfig; @@ -95,8 +96,6 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S boolean _inSystemVM = false; boolean _sslCopy = false; - Random _rand = new Random(System.currentTimeMillis()); - DownloadManager _dlMgr; UploadManager _upldMgr; private String _configSslScr; @@ -219,7 +218,11 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S String nfsHostIp = nfsHostAddr.getHostAddress(); addRouteToInternalIpOrCidr(_localgw, _eth1ip, _eth1mask, nfsHostIp); - return new Answer(cmd, true, "success"); + String nfsPath = nfsHostIp + ":" + uri.getPath(); + String dir = UUID.nameUUIDFromBytes(nfsPath.getBytes()).toString(); + String root = _parent + "/" + dir; + mount(root, nfsPath); + return new SecStorageSetupAnswer(dir); } catch (Exception e) { String msg = "GetRootDir for " + secUrl + " failed due to " + e.toString(); s_logger.error(msg); @@ -707,6 +710,15 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S fillNetworkInformation(cmd); if(_publicIp != null) cmd.setPublicIpAddress(_publicIp); + + Script command = new Script("/bin/bash", s_logger); + command.add("-c"); + command.add("ln -sf " + _parent + " /var/www/html/copy"); + String result = command.execute(); + if (result != null) { + s_logger.warn("Error in linking err=" + result); + return null; + } return new StartupCommand[] {cmd}; } diff --git a/core/src/com/cloud/storage/template/DownloadManagerImpl.java b/core/src/com/cloud/storage/template/DownloadManagerImpl.java index 6bedcc2f99a..17a2fbfd67b 100755 --- a/core/src/com/cloud/storage/template/DownloadManagerImpl.java +++ b/core/src/com/cloud/storage/template/DownloadManagerImpl.java @@ -390,6 +390,9 @@ public class DownloadManagerImpl implements DownloadManager { } File file = _storage.getFile(tmpDir + File.separator + TemplateLocation.Filename); + if ( file.exists() ) { + file.delete(); + } if (!file.createNewFile()) { s_logger.warn("Unable to create new file: " + file.getAbsolutePath()); diff --git a/server/src/com/cloud/storage/StorageManager.java b/server/src/com/cloud/storage/StorageManager.java index 8cbae728cdd..e1210cb5e42 100755 --- a/server/src/com/cloud/storage/StorageManager.java +++ b/server/src/com/cloud/storage/StorageManager.java @@ -193,5 +193,7 @@ public interface StorageManager extends Manager { boolean createStoragePool(long hostId, StoragePoolVO pool); - boolean delPoolFromHost(long hostId); + boolean delPoolFromHost(long hostId); + + HostVO getSecondaryStorageHost(long zoneId, long tmpltId); } diff --git a/server/src/com/cloud/storage/StorageManagerImpl.java b/server/src/com/cloud/storage/StorageManagerImpl.java index a1a423d214b..d1f70f26c65 100755 --- a/server/src/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/com/cloud/storage/StorageManagerImpl.java @@ -940,6 +940,21 @@ public class StorageManagerImpl implements StorageManager, StorageService, Manag return secondaryStorageHost.getStorageUrl(); } + @Override + public HostVO getSecondaryStorageHost(long zoneId, long tmpltId) { + List hosts = _hostDao.listSecondaryStorageHosts(zoneId); + if( hosts == null || hosts.size() == 0) { + return null; + } + for( HostVO host : hosts ) { + VMTemplateHostVO tmpltHost = _vmTemplateHostDao.findByHostTemplate(host.getId(), tmpltId); + if (tmpltHost != null && !tmpltHost.getDestroyed() && tmpltHost.getDownloadState() == VMTemplateStorageResourceAssoc.Status.DOWNLOADED) { + return host; + } + } + return null; + } + @Override public HostVO getSecondaryStorageHost(long zoneId) { List hosts = _hostDao.listSecondaryStorageHosts(zoneId); diff --git a/server/src/com/cloud/storage/download/DownloadMonitor.java b/server/src/com/cloud/storage/download/DownloadMonitor.java index db62749a7c0..1739dcdbb63 100644 --- a/server/src/com/cloud/storage/download/DownloadMonitor.java +++ b/server/src/com/cloud/storage/download/DownloadMonitor.java @@ -40,7 +40,7 @@ public interface DownloadMonitor extends Manager{ public void handleTemplateSync(HostVO host); - public void copyTemplate(VMTemplateVO template, HostVO sourceServer, HostVO destServer) + public boolean copyTemplate(VMTemplateVO template, HostVO sourceServer, HostVO destServer) throws StorageUnavailableException; /*When new host added, take a look at if there are templates needed to be downloaded for the same hypervisor as the host*/ diff --git a/server/src/com/cloud/storage/download/DownloadMonitorImpl.java b/server/src/com/cloud/storage/download/DownloadMonitorImpl.java index ad306d48b6d..969b5b693d8 100755 --- a/server/src/com/cloud/storage/download/DownloadMonitorImpl.java +++ b/server/src/com/cloud/storage/download/DownloadMonitorImpl.java @@ -19,11 +19,9 @@ package com.cloud.storage.download; import java.util.ArrayList; import java.util.Date; -import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Random; import java.util.Set; import java.util.Timer; import java.util.concurrent.ConcurrentHashMap; @@ -42,7 +40,6 @@ import com.cloud.agent.api.storage.DownloadProgressCommand; import com.cloud.agent.api.storage.ListTemplateAnswer; import com.cloud.agent.api.storage.ListTemplateCommand; import com.cloud.agent.api.storage.DownloadProgressCommand.RequestType; -import com.cloud.agent.manager.Commands; import com.cloud.alert.AlertManager; import com.cloud.configuration.dao.ConfigurationDao; import com.cloud.dc.DataCenterVO; @@ -70,6 +67,7 @@ import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VMTemplateHostDao; import com.cloud.storage.dao.VMTemplatePoolDao; import com.cloud.storage.dao.VMTemplateZoneDao; +import com.cloud.storage.secondary.SecondaryStorageVmManager; import com.cloud.storage.template.TemplateConstants; import com.cloud.storage.template.TemplateInfo; import com.cloud.user.Account; @@ -77,7 +75,6 @@ import com.cloud.utils.component.Inject; import com.cloud.utils.db.JoinBuilder; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; -import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.SecondaryStorageVm; import com.cloud.vm.SecondaryStorageVmVO; import com.cloud.vm.UserVmManager; @@ -113,6 +110,7 @@ public class DownloadMonitorImpl implements DownloadMonitor { VMTemplateDao _templateDao = null; @Inject private AgentManager _agentMgr; + @Inject SecondaryStorageVmManager _secMgr; @Inject ConfigurationDao _configDao; @Inject @@ -203,7 +201,7 @@ public class DownloadMonitorImpl implements DownloadMonitor { } @Override - public void copyTemplate(VMTemplateVO template, HostVO sourceServer, HostVO destServer) throws StorageUnavailableException{ + public boolean copyTemplate(VMTemplateVO template, HostVO sourceServer, HostVO destServer) throws StorageUnavailableException{ boolean downloadJobExists = false; VMTemplateHostVO destTmpltHost = null; @@ -213,6 +211,9 @@ public class DownloadMonitorImpl implements DownloadMonitor { if (srcTmpltHost == null) { throw new InvalidParameterValueException("Template " + template.getName() + " not associated with " + sourceServer.getName()); } + if( !_secMgr.generateSetupCommand(sourceServer.getId()) ){ + return false; + } String url = generateCopyUrl(sourceServer, srcTmpltHost); if (url == null) { s_logger.warn("Unable to start/resume copy of template " + template.getUniqueName() + " to " + destServer.getName() + ", no secondary storage vm in running state in source zone"); @@ -232,28 +233,41 @@ public class DownloadMonitorImpl implements DownloadMonitor { if(destTmpltHost != null) { start(); - DownloadCommand dcmd = new DownloadCommand(destServer.getStorageUrl(), url, template, TemplateConstants.DEFAULT_HTTP_AUTH_USER, _copyAuthPasswd, maxTemplateSizeInBytes); - DownloadListener dl = downloadJobExists?_listenerMap.get(destTmpltHost):null; - if (dl == null) { - dl = new DownloadListener(destServer, template, _timer, _vmTemplateHostDao, destTmpltHost.getId(), this, dcmd); - } + DownloadCommand dcmd = + new DownloadCommand(destServer.getStorageUrl(), url, template, TemplateConstants.DEFAULT_HTTP_AUTH_USER, _copyAuthPasswd, maxTemplateSizeInBytes); if (downloadJobExists) { dcmd = new DownloadProgressCommand(dcmd, destTmpltHost.getJobId(), RequestType.GET_OR_RESTART); - dl.setCurrState(destTmpltHost.getDownloadState()); - } - - _listenerMap.put(destTmpltHost, dl); - - long result = _agentMgr.sendToSecStorage(destServer, dcmd, dl); + } + HostVO ssAhost = _agentMgr.getSSAgent(destServer); + if( ssAhost == null ) { + s_logger.warn("There is no secondary storage VM for secondary storage host " + destServer.getName()); + return false; + } + DownloadListener dl = new DownloadListener(ssAhost, template, _timer, _vmTemplateHostDao, destTmpltHost.getId(), this, dcmd); + if (downloadJobExists) { + dl.setCurrState(destTmpltHost.getDownloadState()); + } + DownloadListener old = null; + synchronized (_listenerMap) { + old = _listenerMap.put(destTmpltHost, dl); + } + if( old != null ) { + old.abandon(); + } + + long result = send(ssAhost.getId(), dcmd, dl); if (result == -1) { s_logger.warn("Unable to start /resume COPY of template " + template.getUniqueName() + " to " + destServer.getName()); dl.setDisconnected(); dl.scheduleStatusCheck(RequestType.GET_OR_RESTART); + } else { + return true; } } + return false; } - private String generateCopyUrl(String ipAddress, String path){ + private String generateCopyUrl(String ipAddress, String dir, String path){ String hostname = ipAddress; String scheme = "http"; if (_sslCopy) { @@ -261,7 +275,7 @@ public class DownloadMonitorImpl implements DownloadMonitor { hostname = hostname + ".realhostip.com"; scheme = "https"; } - return scheme + "://" + hostname + "/copy/" + path; + return scheme + "://" + hostname + "/copy/SecStorage/" + dir + "/" + path; } private String generateCopyUrl(HostVO sourceServer, VMTemplateHostVO srcTmpltHost) { @@ -272,7 +286,7 @@ public class DownloadMonitorImpl implements DownloadMonitor { s_logger.warn("A running secondary storage vm has a null public ip?"); return null; } - return generateCopyUrl(ssVm.getPublicIpAddress(), srcTmpltHost.getInstallPath()); + return generateCopyUrl(ssVm.getPublicIpAddress(), sourceServer.getParent(), srcTmpltHost.getInstallPath()); } VMTemplateVO tmplt = _templateDao.findById(srcTmpltHost.getTemplateId()); @@ -297,11 +311,14 @@ public class DownloadMonitorImpl implements DownloadMonitor { } Long maxTemplateSizeInBytes = getMaxTemplateSizeInBytes(); - String url = sserver.getStorageUrl(); + String secUrl = sserver.getStorageUrl(); if(vmTemplateHost != null) { start(); - DownloadCommand dcmd = new DownloadCommand(url, template, maxTemplateSizeInBytes); - dcmd.setUrl(vmTemplateHost.getDownloadUrl()); + DownloadCommand dcmd = + new DownloadCommand(secUrl, template, maxTemplateSizeInBytes); + if (downloadJobExists) { + dcmd = new DownloadProgressCommand(dcmd, vmTemplateHost.getJobId(), RequestType.GET_OR_RESTART); + } if (vmTemplateHost.isCopy()) { dcmd.setCreds(TemplateConstants.DEFAULT_HTTP_AUTH_USER, _copyAuthPasswd); } @@ -312,11 +329,15 @@ public class DownloadMonitorImpl implements DownloadMonitor { } DownloadListener dl = new DownloadListener(ssAhost, template, _timer, _vmTemplateHostDao, vmTemplateHost.getId(), this, dcmd); if (downloadJobExists) { - dcmd = new DownloadProgressCommand(dcmd, vmTemplateHost.getJobId(), RequestType.GET_OR_RESTART); dl.setCurrState(vmTemplateHost.getDownloadState()); } - - _listenerMap.put(vmTemplateHost, dl); + DownloadListener old = null; + synchronized (_listenerMap) { + old = _listenerMap.put(vmTemplateHost, dl); + } + if( old != null ) { + old.abandon(); + } long result = send(ssAhost.getId(), dcmd, dl); if (result == -1) { @@ -361,9 +382,12 @@ public class DownloadMonitorImpl implements DownloadMonitor { public void handleDownloadEvent(HostVO host, VMTemplateVO template, Status dnldStatus) { if ((dnldStatus == VMTemplateStorageResourceAssoc.Status.DOWNLOADED) || (dnldStatus==Status.ABANDONED)){ VMTemplateHostVO vmTemplateHost = new VMTemplateHostVO(host.getId(), template.getId()); - DownloadListener oldListener = _listenerMap.get(vmTemplateHost); + DownloadListener oldListener = null; + synchronized (_listenerMap) { + oldListener = _listenerMap.remove(vmTemplateHost); + } if (oldListener != null) { - _listenerMap.remove(vmTemplateHost); + oldListener.abandon(); } } @@ -666,7 +690,10 @@ public class DownloadMonitorImpl implements DownloadMonitor { _vmTemplateHostDao.listByTemplateStates(templateId, VMTemplateHostVO.Status.DOWNLOAD_IN_PROGRESS, VMTemplateHostVO.Status.NOT_DOWNLOADED); if (downloadsInProgress.size() > 0){ for (VMTemplateHostVO vmthvo: downloadsInProgress) { - DownloadListener dl = _listenerMap.get(vmthvo); + DownloadListener dl = null; + synchronized (_listenerMap) { + dl = _listenerMap.remove(vmthvo); + } if (dl != null) { dl.abandon(); s_logger.info("Stopping download of template " + templateId + " to storage server " + vmthvo.getHostId()); diff --git a/server/src/com/cloud/storage/secondary/SecondaryStorageManagerImpl.java b/server/src/com/cloud/storage/secondary/SecondaryStorageManagerImpl.java index 6cbf696efc2..fad19f75d5c 100644 --- a/server/src/com/cloud/storage/secondary/SecondaryStorageManagerImpl.java +++ b/server/src/com/cloud/storage/secondary/SecondaryStorageManagerImpl.java @@ -34,6 +34,7 @@ import com.cloud.agent.api.Answer; import com.cloud.agent.api.Command; import com.cloud.agent.api.RebootCommand; import com.cloud.agent.api.SecStorageFirewallCfgCommand; +import com.cloud.agent.api.SecStorageSetupAnswer; import com.cloud.agent.api.SecStorageSetupCommand; import com.cloud.agent.api.SecStorageVMSetupCommand; import com.cloud.agent.api.StartupCommand; @@ -246,6 +247,9 @@ public class SecondaryStorageManagerImpl implements SecondaryStorageVmManager, V Answer answer = _agentMgr.easySend(ssHostId, setupCmd); if (answer != null && answer.getResult()) { + SecStorageSetupAnswer an = (SecStorageSetupAnswer) answer; + ssHost.setParent(an.get_dir()); + _hostDao.update(ssHost.getId(), ssHost); if (s_logger.isDebugEnabled()) { s_logger.debug("Successfully programmed secondary storage " + ssHost.getName() + " in secondary storage VM " + secStorageVm.getInstanceName()); } diff --git a/server/src/com/cloud/template/TemplateManager.java b/server/src/com/cloud/template/TemplateManager.java index 742c510a1ff..0c95f4b20d3 100755 --- a/server/src/com/cloud/template/TemplateManager.java +++ b/server/src/com/cloud/template/TemplateManager.java @@ -20,9 +20,11 @@ package com.cloud.template; import java.net.URI; import java.util.List; +import com.cloud.dc.DataCenterVO; import com.cloud.exception.InternalErrorException; import com.cloud.exception.ResourceAllocationException; import com.cloud.exception.StorageUnavailableException; +import com.cloud.host.HostVO; import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.Storage.TemplateType; import com.cloud.storage.StoragePool; @@ -89,31 +91,16 @@ public interface TemplateManager { /** * Copies a template from its current secondary storage server to the secondary storage server in the specified zone. * - * @param templateId - * @param sourceZoneId - * @param destZoneId + * @param template + * @param srcSecHost + * @param srcZone + * @param destZone * @return true if success * @throws InternalErrorException - * URI uri = new URI(url); if ( (uri.getScheme() == null) || (!uri.getScheme().equalsIgnoreCase("ftp") )) { - * throw new IllegalArgumentException("Unsupported scheme for url: " + url); } String host = uri.getHost(); - * - * try { InetAddress hostAddr = InetAddress.getByName(host); if (hostAddr.isAnyLocalAddress() || - * hostAddr.isLinkLocalAddress() || hostAddr.isLoopbackAddress() || hostAddr.isMulticastAddress() ) { throw new - * IllegalArgumentException("Illegal host specified in url"); } if (hostAddr instanceof Inet6Address) { throw - * new IllegalArgumentException("IPV6 addresses not supported (" + hostAddr.getHostAddress() + ")"); } } catch - * (UnknownHostException uhe) { throw new IllegalArgumentException("Unable to resolve " + host); } - * - * if (_dcDao.findById(zoneId) == null) { throw new IllegalArgumentException("Please specify a valid zone."); } - * - * VMTemplateVO template = findTemplateById(templateId); - * - * VMTemplateHostVO tmpltHostRef = findTemplateHostRef(templateId, zoneId); if (tmpltHostRef != null && - * tmpltHostRef.getDownloadState() != com.cloud.storage.VMTemplateStorageResourceAssoc.Status.DOWNLOADED){ throw - * new IllegalArgumentException("The template hasnt been downloaded "); } * @throws StorageUnavailableException * @throws ResourceAllocationException */ - boolean copy(long userId, long templateId, long sourceZoneId, long destZoneId) throws StorageUnavailableException, ResourceAllocationException; + boolean copy(long userId, VMTemplateVO template, HostVO srcSecHost, DataCenterVO srcZone, DataCenterVO dstZone) throws StorageUnavailableException, ResourceAllocationException; /** * Deletes a template from secondary storage servers diff --git a/server/src/com/cloud/template/TemplateManagerImpl.java b/server/src/com/cloud/template/TemplateManagerImpl.java index 78014a6a183..32d9d4355d1 100755 --- a/server/src/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/com/cloud/template/TemplateManagerImpl.java @@ -37,7 +37,6 @@ import com.cloud.agent.api.storage.DestroyCommand; import com.cloud.agent.api.storage.PrimaryStorageDownloadAnswer; import com.cloud.agent.api.storage.PrimaryStorageDownloadCommand; import com.cloud.api.commands.AttachIsoCmd; -import com.cloud.api.commands.CopyIsoCmd; import com.cloud.api.commands.CopyTemplateCmd; import com.cloud.api.commands.DeleteIsoCmd; import com.cloud.api.commands.DeleteTemplateCmd; @@ -94,6 +93,7 @@ import com.cloud.storage.dao.VMTemplatePoolDao; import com.cloud.storage.dao.VMTemplateZoneDao; import com.cloud.storage.dao.VolumeDao; import com.cloud.storage.download.DownloadMonitor; +import com.cloud.storage.secondary.SecondaryStorageVmManager; import com.cloud.storage.upload.UploadMonitor; import com.cloud.template.TemplateAdapter.TemplateAdapterType; import com.cloud.user.Account; @@ -453,50 +453,24 @@ public class TemplateManagerImpl implements TemplateManager, Manager, TemplateSe @Override @DB - public boolean copy(long userId, long templateId, long sourceZoneId, long destZoneId) throws StorageUnavailableException, ResourceAllocationException { - HostVO srcSecHost = _storageMgr.getSecondaryStorageHost(sourceZoneId); - HostVO dstSecHost = _storageMgr.getSecondaryStorageHost(destZoneId); - DataCenterVO destZone = _dcDao.findById(destZoneId); - - DataCenterVO sourceZone = _dcDao.findById(sourceZoneId); - if (sourceZone == null) { - throw new InvalidParameterValueException("Please specify a valid source zone."); - } - - DataCenterVO dstZone = _dcDao.findById(destZoneId); - if (dstZone == null) { - throw new InvalidParameterValueException("Please specify a valid destination zone."); - } - - if (sourceZoneId == destZoneId) { - throw new InvalidParameterValueException("Please specify different source and destination zones."); + public boolean copy(long userId, VMTemplateVO template, HostVO srcSecHost, DataCenterVO srcZone, DataCenterVO dstZone) throws StorageUnavailableException, ResourceAllocationException { + List dstSecHosts = _hostDao.listSecondaryStorageHosts(dstZone.getId()); + long tmpltId = template.getId(); + long dstZoneId = dstZone.getId(); + if (dstSecHosts == null || dstSecHosts.isEmpty() ) { + throw new StorageUnavailableException("Destination zone is not ready", DataCenter.class, dstZone.getId()); } - - if (srcSecHost == null) { - throw new StorageUnavailableException("Source zone is not ready", DataCenter.class, sourceZoneId); - } - if (dstSecHost == null) { - throw new StorageUnavailableException("Destination zone is not ready", DataCenter.class, destZoneId); - } - - VMTemplateVO vmTemplate = _tmpltDao.findById(templateId); - VMTemplateHostVO srcTmpltHost = null; - srcTmpltHost = _tmpltHostDao.findByHostTemplate(srcSecHost.getId(), templateId); - if (srcTmpltHost == null || srcTmpltHost.getDestroyed() || srcTmpltHost.getDownloadState() != VMTemplateStorageResourceAssoc.Status.DOWNLOADED) { - throw new InvalidParameterValueException("Please specify a template that is installed on secondary storage host: " + srcSecHost.getName()); - } - - AccountVO account = _accountDao.findById(vmTemplate.getAccountId()); + AccountVO account = _accountDao.findById(template.getAccountId()); if (_accountMgr.resourceLimitExceeded(account, ResourceType.template)) { ResourceAllocationException rae = new ResourceAllocationException("Maximum number of templates and ISOs for account: " + account.getAccountName() + " has been exceeded."); rae.setResourceType("template"); throw rae; } - + // Event details String copyEventType; String createEventType; - if (vmTemplate.getFormat().equals(ImageFormat.ISO)){ + if (template.getFormat().equals(ImageFormat.ISO)){ copyEventType = EventTypes.EVENT_ISO_COPY; createEventType = EventTypes.EVENT_ISO_CREATE; } else { @@ -507,80 +481,52 @@ public class TemplateManagerImpl implements TemplateManager, Manager, TemplateSe Transaction txn = Transaction.currentTxn(); txn.start(); - - VMTemplateHostVO dstTmpltHost = null; - try { - dstTmpltHost = _tmpltHostDao.findByHostTemplate(dstSecHost.getId(), templateId, true); - if (dstTmpltHost != null) { - dstTmpltHost = _tmpltHostDao.lockRow(dstTmpltHost.getId(), true); - if (dstTmpltHost != null && dstTmpltHost.getDownloadState() == Status.DOWNLOADED) { - if (dstTmpltHost.getDestroyed() == false) { - return true; - } else { - dstTmpltHost.setDestroyed(false); - _tmpltHostDao.update(dstTmpltHost.getId(), dstTmpltHost); - - return true; - } - } else if (dstTmpltHost != null && dstTmpltHost.getDownloadState() == Status.DOWNLOAD_ERROR){ - if (dstTmpltHost.getDestroyed() == true) { - dstTmpltHost.setDestroyed(false); - dstTmpltHost.setDownloadState(Status.NOT_DOWNLOADED); - dstTmpltHost.setDownloadPercent(0); - dstTmpltHost.setCopy(true); - dstTmpltHost.setErrorString(""); - dstTmpltHost.setJobId(null); - _tmpltHostDao.update(dstTmpltHost.getId(), dstTmpltHost); - } - } - } - } finally { - txn.commit(); - } - _tmpltDao.addTemplateToZone(vmTemplate, destZoneId); - _downloadMonitor.copyTemplate(vmTemplate, srcSecHost, dstSecHost); - - if(account.getId() != Account.ACCOUNT_ID_SYSTEM){ - UsageEventVO usageEvent = new UsageEventVO(copyEventType, account.getId(), destZoneId, templateId, null, null, vmTemplate.getSourceTemplateId(), srcTmpltHost.getSize()); - _usageEventDao.persist(usageEvent); - } - return true; + VMTemplateHostVO srcTmpltHost = _tmpltHostDao.findByHostTemplate(srcSecHost.getId(), tmpltId); + for ( HostVO dstSecHost : dstSecHosts ) { + VMTemplateHostVO dstTmpltHost = null; + try { + dstTmpltHost = _tmpltHostDao.findByHostTemplate(dstSecHost.getId(), tmpltId, true); + if (dstTmpltHost != null) { + dstTmpltHost = _tmpltHostDao.lockRow(dstTmpltHost.getId(), true); + if (dstTmpltHost != null && dstTmpltHost.getDownloadState() == Status.DOWNLOADED) { + if (dstTmpltHost.getDestroyed() == false) { + return true; + } else { + dstTmpltHost.setDestroyed(false); + _tmpltHostDao.update(dstTmpltHost.getId(), dstTmpltHost); + + return true; + } + } else if (dstTmpltHost != null && dstTmpltHost.getDownloadState() == Status.DOWNLOAD_ERROR){ + if (dstTmpltHost.getDestroyed() == true) { + dstTmpltHost.setDestroyed(false); + dstTmpltHost.setDownloadState(Status.NOT_DOWNLOADED); + dstTmpltHost.setDownloadPercent(0); + dstTmpltHost.setCopy(true); + dstTmpltHost.setErrorString(""); + dstTmpltHost.setJobId(null); + _tmpltHostDao.update(dstTmpltHost.getId(), dstTmpltHost); + } + } + } + } finally { + txn.commit(); + } + + if(_downloadMonitor.copyTemplate(template, srcSecHost, dstSecHost) ) { + _tmpltDao.addTemplateToZone(template, dstZoneId); + + if(account.getId() != Account.ACCOUNT_ID_SYSTEM){ + UsageEventVO usageEvent = new UsageEventVO(copyEventType, account.getId(), dstZoneId, tmpltId, null, null, null, srcTmpltHost.getSize()); + _usageEventDao.persist(usageEvent); + } + return true; + } + } + return false; } - - @Override - public VirtualMachineTemplate copyIso(CopyIsoCmd cmd) throws StorageUnavailableException, ResourceAllocationException { - Long isoId = cmd.getId(); - Long userId = UserContext.current().getCallerUserId(); - Long sourceZoneId = cmd.getSourceZoneId(); - Long destZoneId = cmd.getDestinationZoneId(); - Account account = UserContext.current().getCaller(); - - //Verify parameters - VMTemplateVO iso = _tmpltDao.findById(isoId); - if (iso == null) { - throw new InvalidParameterValueException("Unable to find ISO with id " + isoId); - } - - boolean isIso = Storage.ImageFormat.ISO.equals(iso.getFormat()); - if (!isIso) { - throw new InvalidParameterValueException("Please specify a valid ISO."); - } - - //Verify account information - String errMsg = "Unable to copy ISO " + isoId; - userId = accountAndUserValidation(account, userId, null, iso, errMsg); - - boolean success = copy(userId, isoId, sourceZoneId, destZoneId); - - VMTemplateVO copiedIso = null; - if (success) { - copiedIso = _tmpltDao.findById(isoId); - } - - return copiedIso; - } - + @Override public VirtualMachineTemplate copyTemplate(CopyTemplateCmd cmd) throws StorageUnavailableException, ResourceAllocationException { @@ -591,28 +537,49 @@ public class TemplateManagerImpl implements TemplateManager, Manager, TemplateSe Account account = UserContext.current().getCaller(); //Verify parameters + + if (sourceZoneId == destZoneId) { + throw new InvalidParameterValueException("Please specify different source and destination zones."); + } + + DataCenterVO sourceZone = _dcDao.findById(sourceZoneId); + if (sourceZone == null) { + throw new InvalidParameterValueException("Please specify a valid source zone."); + } + + DataCenterVO dstZone = _dcDao.findById(destZoneId); + if (dstZone == null) { + throw new InvalidParameterValueException("Please specify a valid destination zone."); + } + VMTemplateVO template = _tmpltDao.findById(templateId); if (template == null) { throw new InvalidParameterValueException("Unable to find template with id"); } - - boolean isIso = Storage.ImageFormat.ISO.equals(template.getFormat()); - if (isIso) { - throw new InvalidParameterValueException("Please specify a valid template."); + + HostVO dstSecHost = _storageMgr.getSecondaryStorageHost(destZoneId, templateId); + if ( dstSecHost != null ) { + s_logger.debug("There is template " + templateId + " in secondary storage " + dstSecHost.getId() + " in zone " + destZoneId + " , don't need to copy"); + return template; } + HostVO srcSecHost = _storageMgr.getSecondaryStorageHost(sourceZoneId, templateId); + if ( srcSecHost == null ) { + throw new InvalidParameterValueException("There is no template " + templateId + " in zone " + sourceZoneId ); + } //Verify account information String errMsg = "Unable to copy template " + templateId; userId = accountAndUserValidation(account, userId, null, template, errMsg); - boolean success = copy(userId, templateId, sourceZoneId, destZoneId); + boolean success = copy(userId, template, srcSecHost, sourceZone, dstZone); - VMTemplateVO copiedTemplate = null; if (success) { - copiedTemplate = _tmpltDao.findById(templateId); + return template; + } else { + s_logger.warn(errMsg); } - return copiedTemplate; + return null; } @Override