mirror of https://github.com/apache/cloudstack.git
Fix metalink download, checksum retry logic and metalink SSVM downloader
This commit is contained in:
nvazquez
Daan Hoogland
parent
1c71a9b867
commit
cdf9ac4679
|
|
@ -38,6 +38,7 @@ public abstract class DirectTemplateDownloaderImpl implements DirectTemplateDown
|
|||
private String downloadedFilePath;
|
||||
private String installPath;
|
||||
private String checksum;
|
||||
private boolean redownload = false;
|
||||
public static final Logger s_logger = Logger.getLogger(DirectTemplateDownloaderImpl.class.getName());
|
||||
|
||||
protected DirectTemplateDownloaderImpl(final String url, final String destPoolPath, final Long templateId, final String checksum) {
|
||||
|
|
@ -92,6 +93,18 @@ public abstract class DirectTemplateDownloaderImpl implements DirectTemplateDown
|
|||
this.downloadedFilePath = filePath;
|
||||
}
|
||||
|
||||
public String getChecksum() {
|
||||
return checksum;
|
||||
}
|
||||
|
||||
public void setChecksum(String checksum) {
|
||||
this.checksum = checksum;
|
||||
}
|
||||
|
||||
public boolean isRedownload() {
|
||||
return redownload;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return filename from url
|
||||
*/
|
||||
|
|
@ -163,7 +176,7 @@ public abstract class DirectTemplateDownloaderImpl implements DirectTemplateDown
|
|||
*/
|
||||
private void resetDownloadFile() {
|
||||
File f = new File(getDownloadedFilePath());
|
||||
s_logger.debug("Resetting download file: " + getDownloadedFilePath() + ", in order to re-download and persist template " + templateId + " on it");
|
||||
s_logger.info("Resetting download file: " + getDownloadedFilePath() + ", in order to re-download and persist template " + templateId + " on it");
|
||||
try {
|
||||
if (f.exists()) {
|
||||
f.delete();
|
||||
|
|
@ -182,15 +195,17 @@ public abstract class DirectTemplateDownloaderImpl implements DirectTemplateDown
|
|||
boolean valid = false;
|
||||
try {
|
||||
while (!valid && retry > 0) {
|
||||
s_logger.debug("Performing checksum validation for downloaded template " + templateId + ", retries left: " + retry);
|
||||
s_logger.info("Performing checksum validation for downloaded template " + templateId + ", retries left: " + retry);
|
||||
valid = DigestHelper.check(checksum, new FileInputStream(downloadedFilePath));
|
||||
retry--;
|
||||
if (!valid && retry > 0) {
|
||||
s_logger.debug("Checksum validation failded, re-downloading template");
|
||||
s_logger.info("Checksum validation failded, re-downloading template");
|
||||
redownload = true;
|
||||
resetDownloadFile();
|
||||
downloadTemplate();
|
||||
}
|
||||
}
|
||||
s_logger.info("Checksum validation for template " + templateId + ": " + (valid ? "succeeded" : "failed"));
|
||||
return valid;
|
||||
} catch (IOException e) {
|
||||
throw new CloudRuntimeException("could not check sum for file: " + downloadedFilePath);
|
||||
|
|
@ -198,6 +213,8 @@ public abstract class DirectTemplateDownloaderImpl implements DirectTemplateDown
|
|||
throw new CloudRuntimeException("Unknown checksum algorithm: " + checksum, e);
|
||||
}
|
||||
}
|
||||
s_logger.info("No checksum provided, skipping checksum validation");
|
||||
return true;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -50,6 +50,7 @@ public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
|
|||
|
||||
public HttpDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum, Map<String, String> headers) {
|
||||
super(url, destPoolPath, templateId, checksum);
|
||||
s_httpClientManager.getParams().setConnectionTimeout(5000);
|
||||
client = new HttpClient(s_httpClientManager);
|
||||
myretryhandler = createRetryTwiceHandler();
|
||||
request = createRequest(url, headers);
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ import com.cloud.utils.script.Script;
|
|||
import org.apache.commons.collections.MapUtils;
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.apache.http.HttpEntity;
|
||||
import org.apache.http.client.config.RequestConfig;
|
||||
import org.apache.http.client.methods.CloseableHttpResponse;
|
||||
import org.apache.http.client.methods.HttpGet;
|
||||
import org.apache.http.client.methods.HttpUriRequest;
|
||||
|
|
@ -61,7 +62,11 @@ public class HttpsDirectTemplateDownloader extends HttpDirectTemplateDownloader
|
|||
throw new CloudRuntimeException("Failure getting SSL context for HTTPS downloader: " + e.getMessage());
|
||||
}
|
||||
SSLConnectionSocketFactory factory = new SSLConnectionSocketFactory(sslcontext, SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
|
||||
httpsClient = HttpClients.custom().setSSLSocketFactory(factory).build();
|
||||
RequestConfig config = RequestConfig.custom()
|
||||
.setConnectTimeout(5000)
|
||||
.setConnectionRequestTimeout(5000)
|
||||
.setSocketTimeout(5000).build();
|
||||
httpsClient = HttpClients.custom().setSSLSocketFactory(factory).setDefaultRequestConfig(config).build();
|
||||
createUriRequest(url, headers);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -19,49 +19,80 @@
|
|||
package com.cloud.agent.direct.download;
|
||||
|
||||
import com.cloud.utils.UriUtils;
|
||||
import com.cloud.utils.exception.CloudRuntimeException;
|
||||
import org.apache.commons.collections.CollectionUtils;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.apache.log4j.Logger;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Random;
|
||||
|
||||
public class MetalinkDirectTemplateDownloader extends HttpDirectTemplateDownloader {
|
||||
|
||||
private String metalinkUrl;
|
||||
private List<String> metalinkUrls;
|
||||
private List<String> metalinkChecksums;
|
||||
private Random random = new Random();
|
||||
private static final Logger s_logger = Logger.getLogger(MetalinkDirectTemplateDownloader.class.getName());
|
||||
|
||||
public MetalinkDirectTemplateDownloader(String url, String destPoolPath, Long templateId, String checksum, Map<String, String> headers) {
|
||||
super(url, templateId, destPoolPath, checksum, headers);
|
||||
metalinkUrl = url;
|
||||
metalinkUrls = UriUtils.getMetalinkUrls(metalinkUrl);
|
||||
metalinkChecksums = UriUtils.getMetalinkChecksums(metalinkUrl);
|
||||
if (CollectionUtils.isEmpty(metalinkUrls)) {
|
||||
throw new CloudRuntimeException("No urls found on metalink file: " + metalinkUrl + ". Not possible to download template " + templateId);
|
||||
}
|
||||
setUrl(metalinkUrls.get(0));
|
||||
s_logger.info("Metalink downloader created, metalink url: " + metalinkUrl + " parsed - " +
|
||||
metalinkUrls.size() + " urls and " +
|
||||
(CollectionUtils.isNotEmpty(metalinkChecksums) ? metalinkChecksums.size() : "0") + " checksums found");
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean downloadTemplate() {
|
||||
s_logger.debug("Retrieving metalink file from: " + getUrl() + " to file: " + getDownloadedFilePath());
|
||||
List<String> metalinkUrls = UriUtils.getMetalinkUrls(getUrl());
|
||||
if (CollectionUtils.isNotEmpty(metalinkUrls)) {
|
||||
String downloadDir = getDirectDownloadTempPath(getTemplateId());
|
||||
boolean downloaded = false;
|
||||
int i = 0;
|
||||
while (!downloaded && i < metalinkUrls.size()) {
|
||||
try {
|
||||
setUrl(metalinkUrls.get(i));
|
||||
s_logger.debug("Trying to download template from metalink url: " + getUrl());
|
||||
File f = new File(getDestPoolPath() + File.separator + downloadDir + File.separator + getFileNameFromUrl());
|
||||
if (f.exists()) {
|
||||
f.delete();
|
||||
f.createNewFile();
|
||||
}
|
||||
setDownloadedFilePath(f.getAbsolutePath());
|
||||
request = createRequest(getUrl(), reqHeaders);
|
||||
downloaded = super.downloadTemplate();
|
||||
if (downloaded) {
|
||||
s_logger.debug("Successfully downloaded template from metalink url: " + getUrl());
|
||||
break;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
s_logger.error("Error downloading template: " + getTemplateId() + " from " + getUrl() + ": " + e.getMessage());
|
||||
}
|
||||
i++;
|
||||
}
|
||||
return downloaded;
|
||||
if (StringUtils.isBlank(getUrl())) {
|
||||
throw new CloudRuntimeException("Download url has not been set, aborting");
|
||||
}
|
||||
return true;
|
||||
String downloadDir = getDirectDownloadTempPath(getTemplateId());
|
||||
boolean downloaded = false;
|
||||
int i = 0;
|
||||
do {
|
||||
if (!isRedownload()) {
|
||||
setUrl(metalinkUrls.get(i));
|
||||
}
|
||||
s_logger.info("Trying to download template from url: " + getUrl());
|
||||
try {
|
||||
File f = new File(getDestPoolPath() + File.separator + downloadDir + File.separator + getFileNameFromUrl());
|
||||
if (f.exists()) {
|
||||
f.delete();
|
||||
f.createNewFile();
|
||||
}
|
||||
setDownloadedFilePath(f.getAbsolutePath());
|
||||
request = createRequest(getUrl(), reqHeaders);
|
||||
downloaded = super.downloadTemplate();
|
||||
if (downloaded) {
|
||||
s_logger.info("Successfully downloaded template from url: " + getUrl());
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
s_logger.error("Error downloading template: " + getTemplateId() + " from " + getUrl() + ": " + e.getMessage());
|
||||
}
|
||||
i++;
|
||||
}
|
||||
while (!downloaded && !isRedownload() && i < metalinkUrls.size());
|
||||
return downloaded;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean validateChecksum() {
|
||||
if (StringUtils.isBlank(getChecksum()) && CollectionUtils.isNotEmpty(metalinkChecksums)) {
|
||||
String chk = metalinkChecksums.get(random.nextInt(metalinkChecksums.size()));
|
||||
setChecksum(chk);
|
||||
s_logger.info("Checksum not provided but " + metalinkChecksums.size() + " found on metalink file, performing checksum using one of them: " + chk);
|
||||
}
|
||||
return super.validateChecksum();
|
||||
}
|
||||
}
|
||||
|
|
@ -549,6 +549,8 @@ public class EventTypes {
|
|||
public static final String EVENT_NIC_SECONDARY_IP_UNASSIGN = "NIC.SECONDARY.IP.UNASSIGN";
|
||||
public static final String EVENT_NIC_SECONDARY_IP_CONFIGURE = "NIC.SECONDARY.IP.CONFIGURE";
|
||||
|
||||
public static final String EVENT_TEMPLATE_DIRECT_DOWNLOAD_FAILURE = "TEMPLATE.DIRECT.DOWNLOAD.FAILURE";
|
||||
public static final String EVENT_ISO_DIRECT_DOWNLOAD_FAILURE = "ISO.DIRECT.DOWNLOAD.FAILURE";
|
||||
|
||||
static {
|
||||
|
||||
|
|
@ -923,6 +925,8 @@ public class EventTypes {
|
|||
entityEventDetails.put(EVENT_NIC_SECONDARY_IP_UNASSIGN, NicSecondaryIp.class);
|
||||
entityEventDetails.put(EVENT_NIC_SECONDARY_IP_CONFIGURE, NicSecondaryIp.class);
|
||||
|
||||
entityEventDetails.put(EVENT_TEMPLATE_DIRECT_DOWNLOAD_FAILURE, VirtualMachineTemplate.class);
|
||||
entityEventDetails.put(EVENT_ISO_DIRECT_DOWNLOAD_FAILURE, "Iso");
|
||||
}
|
||||
|
||||
public static String getEntityForEvent(String eventName) {
|
||||
|
|
|
|||
|
|
@ -20,25 +20,103 @@
|
|||
package com.cloud.storage.template;
|
||||
|
||||
import com.cloud.storage.StorageLayer;
|
||||
import com.cloud.utils.UriUtils;
|
||||
import com.cloud.utils.script.Script;
|
||||
import org.apache.commons.httpclient.HttpClient;
|
||||
import org.apache.commons.httpclient.HttpMethod;
|
||||
import org.apache.commons.httpclient.HttpMethodRetryHandler;
|
||||
import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
|
||||
import org.apache.commons.httpclient.NoHttpResponseException;
|
||||
import org.apache.commons.httpclient.methods.GetMethod;
|
||||
import org.apache.commons.httpclient.params.HttpMethodParams;
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.apache.log4j.Logger;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.OutputStream;
|
||||
import java.util.List;
|
||||
|
||||
public class MetalinkTemplateDownloader extends TemplateDownloaderBase implements TemplateDownloader {
|
||||
|
||||
private TemplateDownloader.Status status = TemplateDownloader.Status.NOT_STARTED;
|
||||
protected HttpClient client;
|
||||
private static final MultiThreadedHttpConnectionManager s_httpClientManager = new MultiThreadedHttpConnectionManager();
|
||||
protected HttpMethodRetryHandler myretryhandler;
|
||||
protected GetMethod request;
|
||||
private boolean toFileSet = false;
|
||||
|
||||
private static final Logger LOGGER = Logger.getLogger(MetalinkTemplateDownloader.class.getName());
|
||||
|
||||
public MetalinkTemplateDownloader(StorageLayer storageLayer, String downloadUrl, String toDir, DownloadCompleteCallback callback, long maxTemplateSize) {
|
||||
super(storageLayer, downloadUrl, toDir, maxTemplateSize, callback);
|
||||
String[] parts = _downloadUrl.split("/");
|
||||
String filename = parts[parts.length - 1];
|
||||
_callback = callback;
|
||||
_toFile = toDir + File.separator + filename;
|
||||
s_httpClientManager.getParams().setConnectionTimeout(5000);
|
||||
client = new HttpClient(s_httpClientManager);
|
||||
myretryhandler = createRetryTwiceHandler();
|
||||
request = createRequest(downloadUrl);
|
||||
}
|
||||
|
||||
protected GetMethod createRequest(String downloadUrl) {
|
||||
GetMethod request = new GetMethod(downloadUrl);
|
||||
request.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, myretryhandler);
|
||||
request.setFollowRedirects(true);
|
||||
if (!toFileSet) {
|
||||
String[] parts = downloadUrl.split("/");
|
||||
String filename = parts[parts.length - 1];
|
||||
_toFile = _toDir + File.separator + filename;
|
||||
toFileSet = true;
|
||||
}
|
||||
return request;
|
||||
}
|
||||
|
||||
protected HttpMethodRetryHandler createRetryTwiceHandler() {
|
||||
return new HttpMethodRetryHandler() {
|
||||
@Override
|
||||
public boolean retryMethod(final HttpMethod method, final IOException exception, int executionCount) {
|
||||
if (executionCount >= 2) {
|
||||
// Do not retry if over max retry count
|
||||
return false;
|
||||
}
|
||||
if (exception instanceof NoHttpResponseException) {
|
||||
// Retry if the server dropped connection on us
|
||||
return true;
|
||||
}
|
||||
if (!method.isRequestSent()) {
|
||||
// Retry if the request has not been sent fully or
|
||||
// if it's OK to retry methods that have been sent
|
||||
return true;
|
||||
}
|
||||
// otherwise do not retry
|
||||
return false;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
private boolean downloadTemplate() {
|
||||
try {
|
||||
client.executeMethod(request);
|
||||
} catch (IOException e) {
|
||||
LOGGER.error("Error on HTTP request: " + e.getMessage());
|
||||
return false;
|
||||
}
|
||||
return performDownload();
|
||||
}
|
||||
|
||||
private boolean performDownload() {
|
||||
try (
|
||||
InputStream in = request.getResponseBodyAsStream();
|
||||
OutputStream out = new FileOutputStream(_toFile);
|
||||
) {
|
||||
IOUtils.copy(in, out);
|
||||
} catch (IOException e) {
|
||||
LOGGER.error("Error downloading template from: " + _downloadUrl + " due to: " + e.getMessage());
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
@Override
|
||||
public long download(boolean resume, DownloadCompleteCallback callback) {
|
||||
if (_status == Status.ABORTED || _status == Status.UNRECOVERABLE_ERROR || _status == Status.DOWNLOAD_FINISHED) {
|
||||
|
|
@ -49,21 +127,26 @@ public class MetalinkTemplateDownloader extends TemplateDownloaderBase implement
|
|||
_start = System.currentTimeMillis();
|
||||
|
||||
status = Status.IN_PROGRESS;
|
||||
Script.runSimpleBashScript("aria2c " + _downloadUrl + " -d " + _toDir);
|
||||
String metalinkFile = _toFile;
|
||||
Script.runSimpleBashScript("rm -f " + metalinkFile);
|
||||
String templateFileName = Script.runSimpleBashScript("ls " + _toDir);
|
||||
String downloadedFile = _toDir + File.separator + templateFileName;
|
||||
_toFile = _toDir + File.separator + "tmpdownld_";
|
||||
Script.runSimpleBashScript("mv " + downloadedFile + " " + _toFile);
|
||||
|
||||
File file = new File(_toFile);
|
||||
if (!file.exists()) {
|
||||
_status = Status.UNRECOVERABLE_ERROR;
|
||||
LOGGER.error("Error downloading template from: " + _downloadUrl);
|
||||
List<String> metalinkUrls = UriUtils.getMetalinkUrls(_downloadUrl);
|
||||
if (CollectionUtils.isEmpty(metalinkUrls)) {
|
||||
LOGGER.error("No URLs found for metalink: " + _downloadUrl);
|
||||
status = Status.UNRECOVERABLE_ERROR;
|
||||
return 0;
|
||||
}
|
||||
_totalBytes = file.length();
|
||||
boolean downloaded = false;
|
||||
int i = 0;
|
||||
while (!downloaded && i < metalinkUrls.size()) {
|
||||
String url = metalinkUrls.get(i);
|
||||
request = createRequest(url);
|
||||
downloaded = downloadTemplate();
|
||||
i++;
|
||||
}
|
||||
if (!downloaded) {
|
||||
LOGGER.error("Template couldnt be downloaded");
|
||||
status = Status.UNRECOVERABLE_ERROR;
|
||||
return 0;
|
||||
}
|
||||
LOGGER.info("Template downloaded successfully on: " + _toFile);
|
||||
status = Status.DOWNLOAD_FINISHED;
|
||||
_downloadTime = System.currentTimeMillis() - _start;
|
||||
if (_callback != null) {
|
||||
|
|
|
|||
|
|
@ -18,8 +18,13 @@
|
|||
//
|
||||
package org.apache.cloudstack.direct.download;
|
||||
|
||||
import static com.cloud.storage.Storage.ImageFormat;
|
||||
|
||||
import com.cloud.agent.AgentManager;
|
||||
import com.cloud.agent.api.Answer;
|
||||
import com.cloud.event.ActionEventUtils;
|
||||
import com.cloud.event.EventTypes;
|
||||
import com.cloud.event.EventVO;
|
||||
import com.cloud.host.Host;
|
||||
import com.cloud.host.HostVO;
|
||||
import com.cloud.host.Status;
|
||||
|
|
@ -52,6 +57,7 @@ import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand;
|
|||
import org.apache.cloudstack.agent.directdownload.SetupDirectDownloadCertificate;
|
||||
import org.apache.cloudstack.agent.directdownload.DirectDownloadAnswer;
|
||||
import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand.DownloadProtocol;
|
||||
import org.apache.cloudstack.context.CallContext;
|
||||
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
|
||||
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
|
||||
import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine;
|
||||
|
|
@ -214,7 +220,7 @@ public class DirectDownloadManagerImpl extends ManagerBase implements DirectDown
|
|||
DownloadProtocol protocol = getProtocolFromUrl(url);
|
||||
DirectDownloadCommand cmd = getDirectDownloadCommandFromProtocol(protocol, url, templateId, to, checksum, headers);
|
||||
|
||||
Answer answer = sendDirectDownloadCommand(cmd, templateId, poolId, host);
|
||||
Answer answer = sendDirectDownloadCommand(cmd, template, poolId, host);
|
||||
|
||||
VMTemplateStoragePoolVO sPoolRef = vmTemplatePoolDao.findByPoolTemplate(poolId, templateId);
|
||||
if (sPoolRef == null) {
|
||||
|
|
@ -237,12 +243,12 @@ public class DirectDownloadManagerImpl extends ManagerBase implements DirectDown
|
|||
* Send direct download command for downloading template with ID templateId on storage pool with ID poolId.<br/>
|
||||
* At first, cmd is sent to host, in case of failure it will retry on other hosts before failing
|
||||
* @param cmd direct download command
|
||||
* @param templateId template id
|
||||
* @param template template
|
||||
* @param poolId pool id
|
||||
* @param host first host to which send the command
|
||||
* @return download answer from any host which could handle cmd
|
||||
*/
|
||||
private Answer sendDirectDownloadCommand(DirectDownloadCommand cmd, long templateId, long poolId, HostVO host) {
|
||||
private Answer sendDirectDownloadCommand(DirectDownloadCommand cmd, VMTemplateVO template, long poolId, HostVO host) {
|
||||
boolean downloaded = false;
|
||||
int retry = 3;
|
||||
Long[] hostsToRetry = getHostsToRetryOn(host.getClusterId(), host.getDataCenterId(), host.getHypervisorType(), host.getId());
|
||||
|
|
@ -257,18 +263,27 @@ public class DirectDownloadManagerImpl extends ManagerBase implements DirectDown
|
|||
retry --;
|
||||
}
|
||||
if (!downloaded) {
|
||||
throw new CloudRuntimeException("Template " + templateId + " could not be downloaded on pool " + poolId + ", failing after trying on several hosts");
|
||||
logUsageEvent(template, poolId);
|
||||
throw new CloudRuntimeException("Template " + template.getId() + " could not be downloaded on pool " + poolId + ", failing after trying on several hosts");
|
||||
}
|
||||
return answer;
|
||||
}
|
||||
|
||||
/**
|
||||
* Log and persist event for direct download failure
|
||||
*/
|
||||
private void logUsageEvent(VMTemplateVO template, long poolId) {
|
||||
String event = EventTypes.EVENT_TEMPLATE_DIRECT_DOWNLOAD_FAILURE;
|
||||
if (template.getFormat() == ImageFormat.ISO) {
|
||||
event = EventTypes.EVENT_ISO_DIRECT_DOWNLOAD_FAILURE;
|
||||
}
|
||||
String description = "Direct Download for template Id: " + template.getId() + "on pool Id: " + poolId + " failed";
|
||||
s_logger.error(description);
|
||||
ActionEventUtils.onCompletedActionEvent(CallContext.current().getCallingUserId(), template.getAccountId(), EventVO.LEVEL_INFO, event, description, 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* Return DirectDownloadCommand according to the protocol
|
||||
* @param protocol
|
||||
* @param url
|
||||
* @param templateId
|
||||
* @param destPool
|
||||
* @return
|
||||
*/
|
||||
private DirectDownloadCommand getDirectDownloadCommandFromProtocol(DownloadProtocol protocol, String url, Long templateId, PrimaryDataStoreTO destPool,
|
||||
String checksum, Map<String, String> httpHeaders) {
|
||||
|
|
|
|||
|
|
@ -294,33 +294,64 @@ public class UriUtils {
|
|||
}
|
||||
|
||||
/**
|
||||
* Get node priority value (if provided), MAX_VALUE if not
|
||||
* Add element to priority list examining node attributes: priority (for urls) and type (for checksums)
|
||||
*/
|
||||
protected static Integer getNodePriority(Node node) {
|
||||
protected static void addPriorityListElementExaminingNode(String tagName, Node node, List<Pair<String, Integer>> priorityList) {
|
||||
Integer priority = Integer.MAX_VALUE;
|
||||
String first = node.getTextContent();
|
||||
if (node.hasAttributes()) {
|
||||
NamedNodeMap attributes = node.getAttributes();
|
||||
for (int k=0; k<attributes.getLength(); k++) {
|
||||
Node attr = attributes.item(k);
|
||||
if (attr.getNodeName().equals("priority")) {
|
||||
if (tagName.equals("url") && attr.getNodeName().equals("priority")) {
|
||||
String prio = attr.getNodeValue().replace("\"", "");
|
||||
return Integer.valueOf(prio);
|
||||
priority = Integer.parseInt(prio);
|
||||
break;
|
||||
} else if (tagName.equals("hash") && attr.getNodeName().equals("type")) {
|
||||
first = "{" + attr.getNodeValue() + "}" + first;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
return Integer.MAX_VALUE;
|
||||
priorityList.add(new Pair<>(first, priority));
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the list of first elements on the list of pairs
|
||||
*/
|
||||
protected static List<String> getListOfFirstElements(List<Pair<String, Integer>> priorityList) {
|
||||
List < String > values = new ArrayList<>();
|
||||
List<String> values = new ArrayList<>();
|
||||
for (Pair<String, Integer> pair : priorityList) {
|
||||
values.add(pair.first());
|
||||
}
|
||||
return values;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return HttpClient with connection timeout
|
||||
*/
|
||||
private static HttpClient getHttpClient() {
|
||||
MultiThreadedHttpConnectionManager s_httpClientManager = new MultiThreadedHttpConnectionManager();
|
||||
s_httpClientManager.getParams().setConnectionTimeout(5000);
|
||||
return new HttpClient(s_httpClientManager);
|
||||
}
|
||||
|
||||
public static List<String> getMetalinkChecksums(String url) {
|
||||
HttpClient httpClient = getHttpClient();
|
||||
GetMethod getMethod = new GetMethod(url);
|
||||
try {
|
||||
if (httpClient.executeMethod(getMethod) == HttpStatus.SC_OK) {
|
||||
InputStream is = getMethod.getResponseBodyAsStream();
|
||||
Map<String, List<String>> checksums = getMultipleValuesFromXML(is, new String[] {"hash"});
|
||||
if (checksums.containsKey("hash")) {
|
||||
return checksums.get("hash");
|
||||
}
|
||||
}
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
/**
|
||||
* Retrieve values from XML documents ordered by ascending priority for each tag name
|
||||
*/
|
||||
|
|
@ -339,8 +370,7 @@ public class UriUtils {
|
|||
List<Pair<String, Integer>> priorityList = new ArrayList<>();
|
||||
for (int j = 0; j < targetNodes.getLength(); j++) {
|
||||
Node node = targetNodes.item(j);
|
||||
Integer priority = getNodePriority(node);
|
||||
priorityList.add(new Pair<>(node.getTextContent(), priority));
|
||||
addPriorityListElementExaminingNode(tagNames[i], node, priorityList);
|
||||
}
|
||||
Collections.sort(priorityList, new Comparator<Pair<String, Integer>>() {
|
||||
@Override
|
||||
|
|
@ -402,7 +432,7 @@ public class UriUtils {
|
|||
* @return a list of Strings containg the URLs of the target locations
|
||||
*/
|
||||
public static List<String> getMetalinkUrls(String metalinkUrl) {
|
||||
HttpClient httpClient = new HttpClient(new MultiThreadedHttpConnectionManager());
|
||||
HttpClient httpClient = getHttpClient();
|
||||
GetMethod getMethod = new GetMethod(metalinkUrl);
|
||||
List<String> urls = new ArrayList<>();
|
||||
try (
|
||||
|
|
@ -424,7 +454,7 @@ public class UriUtils {
|
|||
// use http HEAD method to validate url
|
||||
public static void checkUrlExistence(String url) {
|
||||
if (url.toLowerCase().startsWith("http") || url.toLowerCase().startsWith("https")) {
|
||||
HttpClient httpClient = new HttpClient(new MultiThreadedHttpConnectionManager());
|
||||
HttpClient httpClient = getHttpClient();
|
||||
HeadMethod httphead = new HeadMethod(url);
|
||||
try {
|
||||
if (httpClient.executeMethod(httphead) != HttpStatus.SC_OK) {
|
||||
|
|
@ -434,9 +464,9 @@ public class UriUtils {
|
|||
throw new IllegalArgumentException("Invalid URLs defined on metalink: " + url);
|
||||
}
|
||||
} catch (HttpException hte) {
|
||||
throw new IllegalArgumentException("Cannot reach URL: " + url);
|
||||
throw new IllegalArgumentException("Cannot reach URL: " + url + " due to: " + hte.getMessage());
|
||||
} catch (IOException ioe) {
|
||||
throw new IllegalArgumentException("Cannot reach URL: " + url);
|
||||
throw new IllegalArgumentException("Cannot reach URL: " + url + " due to: " + ioe.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue