From 8e4cec1d87df69db5476e78e82f9efcee7c567ab Mon Sep 17 00:00:00 2001 From: nvazquez Date: Tue, 6 Feb 2018 15:04:28 -0300 Subject: [PATCH] Metalink downloader rework, retrying logic and refactoring --- .../DirectTemplateDownloaderImpl.java | 37 ++++++++- .../HttpDirectTemplateDownloader.java | 6 +- .../HttpsDirectTemplateDownloader.java | 17 +++-- .../MetalinkDirectTemplateDownloader.java | 52 ++++++++----- .../directdownload/DirectDownloadCommand.java | 10 ++- .../HttpDirectDownloadCommand.java | 9 +-- .../HttpsDirectDownloadCommand.java | 2 +- .../MetalinkDirectDownloadCommand.java | 6 +- .../NfsDirectDownloadCommand.java | 6 +- .../src/com/cloud/host/dao/HostDao.java | 3 + .../src/com/cloud/host/dao/HostDaoImpl.java | 33 ++++++++ .../kvm/storage/KVMStorageProcessor.java | 6 +- .../download/DirectDownloadManagerImpl.java | 75 +++++++++++++++++-- utils/src/com/cloud/utils/UriUtils.java | 71 +++++++++++++++--- 14 files changed, 271 insertions(+), 62 deletions(-) diff --git a/agent/src/com/cloud/agent/direct/download/DirectTemplateDownloaderImpl.java b/agent/src/com/cloud/agent/direct/download/DirectTemplateDownloaderImpl.java index f9963615e4b..285346b6eed 100644 --- a/agent/src/com/cloud/agent/direct/download/DirectTemplateDownloaderImpl.java +++ b/agent/src/com/cloud/agent/direct/download/DirectTemplateDownloaderImpl.java @@ -28,6 +28,7 @@ import java.io.FileInputStream; import java.io.IOException; import java.security.NoSuchAlgorithmException; import java.util.UUID; +import org.apache.log4j.Logger; public abstract class DirectTemplateDownloaderImpl implements DirectTemplateDownloader { @@ -37,6 +38,7 @@ public abstract class DirectTemplateDownloaderImpl implements DirectTemplateDown private String downloadedFilePath; private String installPath; private String checksum; + public static final Logger s_logger = Logger.getLogger(DirectTemplateDownloaderImpl.class.getName()); protected DirectTemplateDownloaderImpl(final String url, final String destPoolPath, final Long templateId, final String checksum) { this.url = url; @@ -70,6 +72,10 @@ public abstract class DirectTemplateDownloaderImpl implements DirectTemplateDown return url; } + public void setUrl(String url) { + this.url = url; + } + public String getDestPoolPath() { return destPoolPath; } @@ -152,11 +158,40 @@ public abstract class DirectTemplateDownloaderImpl implements DirectTemplateDown return new DirectTemplateInformation(installPath, size, checksum); } + /** + * Delete and create download file + */ + private void resetDownloadFile() { + File f = new File(getDownloadedFilePath()); + s_logger.debug("Resetting download file: " + getDownloadedFilePath() + ", in order to re-download and persist template " + templateId + " on it"); + try { + if (f.exists()) { + f.delete(); + } + f.createNewFile(); + } catch (IOException e) { + s_logger.error("Error creating file to download on: " + getDownloadedFilePath() + " due to: " + e.getMessage()); + throw new CloudRuntimeException("Failed to create download file for direct download"); + } + } + @Override public boolean validateChecksum() { if (StringUtils.isNotBlank(checksum)) { + int retry = 3; + boolean valid = false; try { - return DigestHelper.check(checksum, new FileInputStream(downloadedFilePath)); + while (!valid && retry > 0) { + s_logger.debug("Performing checksum validation for downloaded template " + templateId + ", retries left: " + retry); + valid = DigestHelper.check(checksum, new FileInputStream(downloadedFilePath)); + retry--; + if (!valid && retry > 0) { + s_logger.debug("Checksum validation failded, re-downloading template"); + resetDownloadFile(); + downloadTemplate(); + } + } + return valid; } catch (IOException e) { throw new CloudRuntimeException("could not check sum for file: " + downloadedFilePath); } catch (NoSuchAlgorithmException e) { diff --git a/agent/src/com/cloud/agent/direct/download/HttpDirectTemplateDownloader.java b/agent/src/com/cloud/agent/direct/download/HttpDirectTemplateDownloader.java index afb47708656..e2b74c29ac8 100644 --- a/agent/src/com/cloud/agent/direct/download/HttpDirectTemplateDownloader.java +++ b/agent/src/com/cloud/agent/direct/download/HttpDirectTemplateDownloader.java @@ -36,16 +36,17 @@ import java.io.FileOutputStream; import java.io.InputStream; import java.io.IOException; import java.io.OutputStream; +import java.util.HashMap; import java.util.Map; public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl { - private HttpClient client; + protected HttpClient client; private static final MultiThreadedHttpConnectionManager s_httpClientManager = new MultiThreadedHttpConnectionManager(); - private static final int CHUNK_SIZE = 1024 * 1024; //1M protected HttpMethodRetryHandler myretryhandler; public static final Logger s_logger = Logger.getLogger(HttpDirectTemplateDownloader.class.getName()); protected GetMethod request; + protected Map reqHeaders = new HashMap<>(); public HttpDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum, Map headers) { super(url, destPoolPath, templateId, checksum); @@ -69,6 +70,7 @@ public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl { if (MapUtils.isNotEmpty(headers)) { for (String key : headers.keySet()) { request.setRequestHeader(key, headers.get(key)); + reqHeaders.put(key, headers.get(key)); } } return request; diff --git a/agent/src/com/cloud/agent/direct/download/HttpsDirectTemplateDownloader.java b/agent/src/com/cloud/agent/direct/download/HttpsDirectTemplateDownloader.java index 98c223ef827..665f1a10a83 100644 --- a/agent/src/com/cloud/agent/direct/download/HttpsDirectTemplateDownloader.java +++ b/agent/src/com/cloud/agent/direct/download/HttpsDirectTemplateDownloader.java @@ -21,6 +21,7 @@ package com.cloud.agent.direct.download; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.script.Script; +import org.apache.commons.collections.MapUtils; import org.apache.commons.io.IOUtils; import org.apache.http.HttpEntity; import org.apache.http.client.methods.CloseableHttpResponse; @@ -44,14 +45,15 @@ import java.security.KeyStore; import java.security.KeyStoreException; import java.security.NoSuchAlgorithmException; import java.security.cert.CertificateException; +import java.util.Map; public class HttpsDirectTemplateDownloader extends HttpDirectTemplateDownloader { private CloseableHttpClient httpsClient; private HttpUriRequest req; - public HttpsDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum) { - super(url, templateId, destPoolPath, checksum, null); + public HttpsDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum, Map headers) { + super(url, templateId, destPoolPath, checksum, headers); SSLContext sslcontext = null; try { sslcontext = getSSLContext(); @@ -60,11 +62,16 @@ public class HttpsDirectTemplateDownloader extends HttpDirectTemplateDownloader } SSLConnectionSocketFactory factory = new SSLConnectionSocketFactory(sslcontext, SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER); httpsClient = HttpClients.custom().setSSLSocketFactory(factory).build(); - req = createUriRequest(url); + createUriRequest(url, headers); } - protected HttpUriRequest createUriRequest(String downloadUrl) { - return new HttpGet(downloadUrl); + protected void createUriRequest(String downloadUrl, Map headers) { + req = new HttpGet(downloadUrl); + if (MapUtils.isNotEmpty(headers)) { + for (String headerKey: headers.keySet()) { + req.setHeader(headerKey, headers.get(headerKey)); + } + } } private SSLContext getSSLContext() throws KeyStoreException, NoSuchAlgorithmException, CertificateException, IOException, KeyManagementException { diff --git a/agent/src/com/cloud/agent/direct/download/MetalinkDirectTemplateDownloader.java b/agent/src/com/cloud/agent/direct/download/MetalinkDirectTemplateDownloader.java index 016c26c77a9..dae45cefa5e 100644 --- a/agent/src/com/cloud/agent/direct/download/MetalinkDirectTemplateDownloader.java +++ b/agent/src/com/cloud/agent/direct/download/MetalinkDirectTemplateDownloader.java @@ -18,32 +18,50 @@ // package com.cloud.agent.direct.download; -import com.cloud.utils.script.Script; +import com.cloud.utils.UriUtils; +import org.apache.commons.collections.CollectionUtils; import java.io.File; +import java.util.List; +import java.util.Map; -public class MetalinkDirectTemplateDownloader extends DirectTemplateDownloaderImpl { +public class MetalinkDirectTemplateDownloader extends HttpDirectTemplateDownloader { - private String downloadDir; - - public MetalinkDirectTemplateDownloader(String url, String destPoolPath, Long templateId, String checksum) { - super(url, destPoolPath, templateId, checksum); - String relativeDir = getDirectDownloadTempPath(templateId); - downloadDir = getDestPoolPath() + File.separator + relativeDir; - createFolder(downloadDir); + public MetalinkDirectTemplateDownloader(String url, String destPoolPath, Long templateId, String checksum, Map headers) { + super(url, templateId, destPoolPath, checksum, headers); } @Override public boolean downloadTemplate() { - String downloadCommand = "aria2c " + getUrl() + " -d " + downloadDir + " --check-integrity=true"; - Script.runSimpleBashScript(downloadCommand); - //Remove .metalink file - Script.runSimpleBashScript("rm -f " + downloadDir + File.separator + getFileNameFromUrl()); - String fileName = Script.runSimpleBashScript("ls " + downloadDir); - if (fileName == null) { - return false; + s_logger.debug("Retrieving metalink file from: " + getUrl() + " to file: " + getDownloadedFilePath()); + List metalinkUrls = UriUtils.getMetalinkUrls(getUrl()); + if (CollectionUtils.isNotEmpty(metalinkUrls)) { + String downloadDir = getDirectDownloadTempPath(getTemplateId()); + boolean downloaded = false; + int i = 0; + while (!downloaded && i < metalinkUrls.size()) { + try { + setUrl(metalinkUrls.get(i)); + s_logger.debug("Trying to download template from metalink url: " + getUrl()); + File f = new File(getDestPoolPath() + File.separator + downloadDir + File.separator + getFileNameFromUrl()); + if (f.exists()) { + f.delete(); + f.createNewFile(); + } + setDownloadedFilePath(f.getAbsolutePath()); + request = createRequest(getUrl(), reqHeaders); + downloaded = super.downloadTemplate(); + if (downloaded) { + s_logger.debug("Successfully downloaded template from metalink url: " + getUrl()); + break; + } + } catch (Exception e) { + s_logger.error("Error downloading template: " + getTemplateId() + " from " + getUrl() + ": " + e.getMessage()); + } + i++; + } + return downloaded; } - setDownloadedFilePath(downloadDir + File.separator + fileName); return true; } } \ No newline at end of file diff --git a/core/src/org/apache/cloudstack/agent/directdownload/DirectDownloadCommand.java b/core/src/org/apache/cloudstack/agent/directdownload/DirectDownloadCommand.java index dd4e2c200ad..d9fb8e23293 100644 --- a/core/src/org/apache/cloudstack/agent/directdownload/DirectDownloadCommand.java +++ b/core/src/org/apache/cloudstack/agent/directdownload/DirectDownloadCommand.java @@ -23,6 +23,8 @@ import com.cloud.agent.api.Command; import org.apache.cloudstack.storage.command.StorageSubSystemCommand; import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; +import java.util.Map; + public abstract class DirectDownloadCommand extends Command implements StorageSubSystemCommand { public enum DownloadProtocol { @@ -33,12 +35,14 @@ public abstract class DirectDownloadCommand extends Command implements StorageSu private Long templateId; private PrimaryDataStoreTO destPool; private String checksum; + private Map headers; - protected DirectDownloadCommand (final String url, final Long templateId, final PrimaryDataStoreTO destPool, final String checksum) { + protected DirectDownloadCommand (final String url, final Long templateId, final PrimaryDataStoreTO destPool, final String checksum, final Map headers) { this.url = url; this.templateId = templateId; this.destPool = destPool; this.checksum = checksum; + this.headers = headers; } public String getUrl() { @@ -57,6 +61,10 @@ public abstract class DirectDownloadCommand extends Command implements StorageSu return checksum; } + public Map getHeaders() { + return headers; + } + @Override public void setExecuteInSequence(boolean inSeq) { } diff --git a/core/src/org/apache/cloudstack/agent/directdownload/HttpDirectDownloadCommand.java b/core/src/org/apache/cloudstack/agent/directdownload/HttpDirectDownloadCommand.java index 1688b8e2e5d..6748988ebed 100644 --- a/core/src/org/apache/cloudstack/agent/directdownload/HttpDirectDownloadCommand.java +++ b/core/src/org/apache/cloudstack/agent/directdownload/HttpDirectDownloadCommand.java @@ -25,15 +25,8 @@ import java.util.Map; public class HttpDirectDownloadCommand extends DirectDownloadCommand { - private Map headers; - public HttpDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum, Map headers) { - super(url, templateId, destPool, checksum); - this.headers = headers; - } - - public Map getHeaders() { - return headers; + super(url, templateId, destPool, checksum, headers); } } \ No newline at end of file diff --git a/core/src/org/apache/cloudstack/agent/directdownload/HttpsDirectDownloadCommand.java b/core/src/org/apache/cloudstack/agent/directdownload/HttpsDirectDownloadCommand.java index b4032fdb9d4..278b90b1d9a 100644 --- a/core/src/org/apache/cloudstack/agent/directdownload/HttpsDirectDownloadCommand.java +++ b/core/src/org/apache/cloudstack/agent/directdownload/HttpsDirectDownloadCommand.java @@ -26,6 +26,6 @@ import java.util.Map; public class HttpsDirectDownloadCommand extends DirectDownloadCommand { public HttpsDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum, Map headers) { - super(url, templateId, destPool, checksum); + super(url, templateId, destPool, checksum, headers); } } diff --git a/core/src/org/apache/cloudstack/agent/directdownload/MetalinkDirectDownloadCommand.java b/core/src/org/apache/cloudstack/agent/directdownload/MetalinkDirectDownloadCommand.java index 073eb42a9b9..b3c81069a71 100644 --- a/core/src/org/apache/cloudstack/agent/directdownload/MetalinkDirectDownloadCommand.java +++ b/core/src/org/apache/cloudstack/agent/directdownload/MetalinkDirectDownloadCommand.java @@ -21,10 +21,12 @@ package org.apache.cloudstack.agent.directdownload; import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; +import java.util.Map; + public class MetalinkDirectDownloadCommand extends DirectDownloadCommand { - public MetalinkDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum) { - super(url, templateId, destPool, checksum); + public MetalinkDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum, Map headers) { + super(url, templateId, destPool, checksum, headers); } } \ No newline at end of file diff --git a/core/src/org/apache/cloudstack/agent/directdownload/NfsDirectDownloadCommand.java b/core/src/org/apache/cloudstack/agent/directdownload/NfsDirectDownloadCommand.java index 44b154bf462..bbd49a76a34 100644 --- a/core/src/org/apache/cloudstack/agent/directdownload/NfsDirectDownloadCommand.java +++ b/core/src/org/apache/cloudstack/agent/directdownload/NfsDirectDownloadCommand.java @@ -21,10 +21,12 @@ package org.apache.cloudstack.agent.directdownload; import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; +import java.util.Map; + public class NfsDirectDownloadCommand extends DirectDownloadCommand { - public NfsDirectDownloadCommand(final String url, final Long templateId, final PrimaryDataStoreTO destPool, final String checksum) { - super(url, templateId, destPool, checksum); + public NfsDirectDownloadCommand(final String url, final Long templateId, final PrimaryDataStoreTO destPool, final String checksum, final Map headers) { + super(url, templateId, destPool, checksum, headers); } } \ No newline at end of file diff --git a/engine/schema/src/com/cloud/host/dao/HostDao.java b/engine/schema/src/com/cloud/host/dao/HostDao.java index c7d84d64e0f..00cf8c59461 100755 --- a/engine/schema/src/com/cloud/host/dao/HostDao.java +++ b/engine/schema/src/com/cloud/host/dao/HostDao.java @@ -23,6 +23,7 @@ import com.cloud.host.Host; import com.cloud.host.Host.Type; import com.cloud.host.HostVO; import com.cloud.host.Status; +import com.cloud.hypervisor.Hypervisor; import com.cloud.info.RunningHostCountInfo; import com.cloud.resource.ResourceState; import com.cloud.utils.db.GenericDao; @@ -96,4 +97,6 @@ public interface HostDao extends GenericDao, StateDao listByDataCenterIdAndHypervisorType(long zoneId, Hypervisor.HypervisorType hypervisorType); } diff --git a/engine/schema/src/com/cloud/host/dao/HostDaoImpl.java b/engine/schema/src/com/cloud/host/dao/HostDaoImpl.java index 1b6935915ff..ff8fdc8de3e 100755 --- a/engine/schema/src/com/cloud/host/dao/HostDaoImpl.java +++ b/engine/schema/src/com/cloud/host/dao/HostDaoImpl.java @@ -31,6 +31,8 @@ import javax.ejb.Local; import javax.inject.Inject; import javax.persistence.TableGenerator; +import com.cloud.hypervisor.Hypervisor; +import com.cloud.org.Grouping; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -427,6 +429,37 @@ public class HostDaoImpl extends GenericDaoBase implements HostDao return listBy(sc); } + @Override + public List listByDataCenterIdAndHypervisorType(long zoneId, Hypervisor.HypervisorType hypervisorType) { + SearchBuilder clusterSearch = _clusterDao.createSearchBuilder(); + + clusterSearch.and("allocationState", clusterSearch.entity().getAllocationState(), SearchCriteria.Op.EQ); + clusterSearch.and("hypervisorType", clusterSearch.entity().getHypervisorType(), SearchCriteria.Op.EQ); + + SearchBuilder hostSearch = createSearchBuilder(); + + hostSearch.and("dc", hostSearch.entity().getDataCenterId(), SearchCriteria.Op.EQ); + hostSearch.and("type", hostSearch.entity().getType(), Op.EQ); + hostSearch.and("status", hostSearch.entity().getStatus(), Op.EQ); + hostSearch.and("resourceState", hostSearch.entity().getResourceState(), Op.EQ); + + hostSearch.join("clusterSearch", clusterSearch, hostSearch.entity().getClusterId(), clusterSearch.entity().getId(), JoinBuilder.JoinType.INNER); + + hostSearch.done(); + + SearchCriteria sc = hostSearch.create(); + + sc.setParameters("dc", zoneId); + sc.setParameters("type", Host.Type.Routing); + sc.setParameters("status", Status.Up); + sc.setParameters("resourceState", ResourceState.Enabled); + + sc.setJoinParameters("clusterSearch", "allocationState", Grouping.AllocationState.Enabled); + sc.setJoinParameters("clusterSearch", "hypervisorType", hypervisorType.toString()); + + return listBy(sc); + } + @Override public HostVO findByGuid(String guid) { SearchCriteria sc = GuidSearch.create("guid", guid); diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java index 9238684d231..3f37fd314f2 100644 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java @@ -1312,13 +1312,13 @@ public class KVMStorageProcessor implements StorageProcessor { DirectTemplateDownloader downloader; if (cmd instanceof HttpDirectDownloadCommand) { - downloader = new HttpDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPool.getLocalPath(), cmd.getChecksum(), ((HttpDirectDownloadCommand) cmd).getHeaders()); + downloader = new HttpDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPool.getLocalPath(), cmd.getChecksum(), cmd.getHeaders()); } else if (cmd instanceof HttpsDirectDownloadCommand) { - downloader = new HttpsDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPool.getLocalPath(), cmd.getChecksum()); + downloader = new HttpsDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPool.getLocalPath(), cmd.getChecksum(), cmd.getHeaders()); } else if (cmd instanceof NfsDirectDownloadCommand) { downloader = new NfsDirectTemplateDownloader(cmd.getUrl(), destPool.getLocalPath(), cmd.getTemplateId(), cmd.getChecksum()); } else if (cmd instanceof MetalinkDirectDownloadCommand) { - downloader = new MetalinkDirectTemplateDownloader(cmd.getUrl(), destPool.getLocalPath(), cmd.getTemplateId(), cmd.getChecksum()); + downloader = new MetalinkDirectTemplateDownloader(cmd.getUrl(), destPool.getLocalPath(), cmd.getTemplateId(), cmd.getChecksum(), cmd.getHeaders()); } else { return new DirectDownloadAnswer(false, "Unsupported protocol, please provide HTTP(S), NFS or a metalink"); } diff --git a/server/src/org/apache/cloudstack/direct/download/DirectDownloadManagerImpl.java b/server/src/org/apache/cloudstack/direct/download/DirectDownloadManagerImpl.java index 5d0ca37c324..40aba6f2bfe 100644 --- a/server/src/org/apache/cloudstack/direct/download/DirectDownloadManagerImpl.java +++ b/server/src/org/apache/cloudstack/direct/download/DirectDownloadManagerImpl.java @@ -24,7 +24,7 @@ import com.cloud.host.Host; import com.cloud.host.HostVO; import com.cloud.host.Status; import com.cloud.host.dao.HostDao; -import com.cloud.hypervisor.Hypervisor; +import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.storage.DataStoreRole; import com.cloud.storage.VMTemplateStoragePoolVO; import com.cloud.storage.VMTemplateStorageResourceAssoc; @@ -40,6 +40,8 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Arrays; +import java.util.Collections; import javax.inject.Inject; import org.apache.cloudstack.agent.directdownload.HttpsDirectDownloadCommand; @@ -57,6 +59,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStore; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.log4j.Logger; @@ -136,6 +139,50 @@ public class DirectDownloadManagerImpl extends ManagerBase implements DirectDown return headers; } + /** + * Get running host IDs within the same hypervisor, cluster and datacenter than hostId. ID hostId is not included on the returned list + */ + protected List getRunningHostIdsInTheSameCluster(Long clusterId, long dataCenterId, HypervisorType hypervisorType, long hostId) { + List hosts = hostDao.listByDataCenterIdAndHypervisorType(dataCenterId, hypervisorType); + List list = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(list)) { + for (HostVO host : hosts) { + if (host.getHypervisorType().equals(hypervisorType) && host.getStatus().equals(Status.Up) && + host.getType().equals(Host.Type.Routing) && host.getClusterId().equals(clusterId) && + host.getId() != hostId) { + list.add(host.getId()); + } + } + } + Collections.shuffle(list); + return list; + } + + /** + * Create host IDs array having hostId as the first element + */ + protected Long[] createHostIdsList(List hostIds, long hostId) { + if (CollectionUtils.isEmpty(hostIds)) { + return Arrays.asList(hostId).toArray(new Long[1]); + } + Long[] ids = new Long[hostIds.size() + 1]; + ids[0] = hostId; + int i = 1; + for (Long id : hostIds) { + ids[i] = id; + i++; + } + return ids; + } + + /** + * Get hosts to retry download having hostId as the first element + */ + protected Long[] getHostsToRetryOn(Long clusterId, long dataCenterId, HypervisorType hypervisorType, long hostId) { + List hostIds = getRunningHostIdsInTheSameCluster(clusterId, dataCenterId, hypervisorType, hostId); + return createHostIdsList(hostIds, hostId); + } + @Override public void downloadTemplate(long templateId, long poolId, long hostId) { VMTemplateVO template = vmTemplateDao.findById(templateId); @@ -166,10 +213,22 @@ public class DirectDownloadManagerImpl extends ManagerBase implements DirectDown DownloadProtocol protocol = getProtocolFromUrl(url); DirectDownloadCommand cmd = getDirectDownloadCommandFromProtocol(protocol, url, templateId, to, checksum, headers); - Answer answer = agentManager.easySend(hostId, cmd); - if (answer == null || !answer.getResult()) { - throw new CloudRuntimeException("Host " + hostId + " could not download template " + - templateId + " on pool " + poolId); + + boolean downloaded = false; + int retry = 3; + Long[] hostsToRetry = getHostsToRetryOn(host.getClusterId(), host.getDataCenterId(), host.getHypervisorType(), hostId); + int hostIndex = 0; + Answer answer = null; + Long hostToSendDownloadCmd = hostsToRetry[hostIndex]; + while (!downloaded && retry > 0) { + s_logger.debug("Sending Direct download command to host " + hostToSendDownloadCmd); + answer = agentManager.easySend(hostToSendDownloadCmd, cmd); + downloaded = answer != null && answer.getResult(); + hostToSendDownloadCmd = hostsToRetry[(hostIndex + 1) % hostsToRetry.length]; + retry --; + } + if (!downloaded) { + throw new CloudRuntimeException("Template " + templateId + " could not be downloaded on pool " + poolId + ", failing after trying on several hosts"); } VMTemplateStoragePoolVO sPoolRef = vmTemplatePoolDao.findByPoolTemplate(poolId, templateId); @@ -204,9 +263,9 @@ public class DirectDownloadManagerImpl extends ManagerBase implements DirectDown } else if (protocol.equals(DownloadProtocol.HTTPS)) { return new HttpsDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders); } else if (protocol.equals(DownloadProtocol.NFS)) { - return new NfsDirectDownloadCommand(url, templateId, destPool, checksum); + return new NfsDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders); } else if (protocol.equals(DownloadProtocol.METALINK)) { - return new MetalinkDirectDownloadCommand(url, templateId, destPool, checksum); + return new MetalinkDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders); } else { return null; } @@ -218,7 +277,7 @@ public class DirectDownloadManagerImpl extends ManagerBase implements DirectDown List hosts = new ArrayList<>(); for (HostVO host: hostsQuery) { if (host.getStatus().equals(Status.Up) && - host.getHypervisorType().equals(Hypervisor.HypervisorType.KVM)) { + host.getHypervisorType().equals(HypervisorType.KVM)) { hosts.add(host); } } diff --git a/utils/src/com/cloud/utils/UriUtils.java b/utils/src/com/cloud/utils/UriUtils.java index bec93b10616..5ec2a626c6a 100644 --- a/utils/src/com/cloud/utils/UriUtils.java +++ b/utils/src/com/cloud/utils/UriUtils.java @@ -19,14 +19,22 @@ package com.cloud.utils; +import java.util.ArrayList; +import java.util.List; +import java.util.ListIterator; +import java.util.StringTokenizer; +import java.util.Collections; +import java.util.Comparator; import java.util.Map; import java.util.HashMap; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; + import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.Node; import org.w3c.dom.NodeList; +import org.w3c.dom.NamedNodeMap; import java.io.File; import java.io.IOException; import java.io.InputStream; @@ -37,10 +45,6 @@ import java.net.URI; import java.net.URISyntaxException; import java.net.URLEncoder; import java.net.UnknownHostException; -import java.util.ArrayList; -import java.util.List; -import java.util.ListIterator; -import java.util.StringTokenizer; import javax.net.ssl.HttpsURLConnection; @@ -289,6 +293,37 @@ public class UriUtils { } } + /** + * Get node priority value (if provided), MAX_VALUE if not + */ + protected static Integer getNodePriority(Node node) { + if (node.hasAttributes()) { + NamedNodeMap attributes = node.getAttributes(); + for (int k=0; k getListOfFirstElements(List> priorityList) { + List < String > values = new ArrayList<>(); + for (Pair pair : priorityList) { + values.add(pair.first()); + } + return values; + } + + /** + * Retrieve values from XML documents ordered by ascending priority for each tag name + */ protected static Map> getMultipleValuesFromXML(InputStream is, String[] tagNames) { Map> returnValues = new HashMap>(); try { @@ -299,14 +334,21 @@ public class UriUtils { for (int i = 0; i < tagNames.length; i++) { NodeList targetNodes = rootElement.getElementsByTagName(tagNames[i]); if (targetNodes.getLength() <= 0) { - s_logger.error("no " + tagNames[i] + " tag in XML response...returning null"); + s_logger.error("no " + tagNames[i] + " tag in XML response..."); } else { - List valueList = new ArrayList(); + List> priorityList = new ArrayList<>(); for (int j = 0; j < targetNodes.getLength(); j++) { Node node = targetNodes.item(j); - valueList.add(node.getTextContent()); + Integer priority = getNodePriority(node); + priorityList.add(new Pair<>(node.getTextContent(), priority)); } - returnValues.put(tagNames[i], valueList); + Collections.sort(priorityList, new Comparator>() { + @Override + public int compare(Pair x, Pair y) { + return x.second().compareTo(y.second()); + } + }); + returnValues.put(tagNames[i], getListOfFirstElements(priorityList)); } } } catch (Exception ex) { @@ -353,18 +395,23 @@ public class UriUtils { } /** - * Get list of urls on metalink - * @param metalinkUrl - * @return + * Get list of urls on metalink ordered by ascending priority (for those which priority tag is not defined, highest priority value is assumed) */ public static List getMetalinkUrls(String metalinkUrl) { HttpClient httpClient = new HttpClient(new MultiThreadedHttpConnectionManager()); GetMethod getMethod = new GetMethod(metalinkUrl); List urls = new ArrayList<>(); + int status; + try { + status = httpClient.executeMethod(getMethod); + } catch (IOException e) { + s_logger.error("Error retrieving urls form metalink: " + metalinkUrl); + return null; + } try ( InputStream is = getMethod.getResponseBodyAsStream() ) { - if (httpClient.executeMethod(getMethod) == HttpStatus.SC_OK) { + if (status == HttpStatus.SC_OK) { Map> metalinkUrlsMap = getMultipleValuesFromXML(is, new String[] {"url"}); if (metalinkUrlsMap.containsKey("url")) { List metalinkUrls = metalinkUrlsMap.get("url");