etan39
/
cloudstack
mirror of https://github.com/apache/cloudstack.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Metalink downloader rework, retrying logic and refactoring

This commit is contained in:
nvazquez 2018-02-06 15:04:28 -03:00
parent ced27374a8
commit 8e4cec1d87
14 changed files with 271 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
agent/src/com/cloud/agent/direct/download/DirectTemplateDownloaderImpl.java
View File

@ -28,6 +28,7 @@ import java.io.FileInputStream;
import java.io.IOException;
import java.security.NoSuchAlgorithmException;
import java.util.UUID;
import org.apache.log4j.Logger;
public abstract class DirectTemplateDownloaderImpl implements DirectTemplateDownloader {
@ -37,6 +38,7 @@ public abstract class DirectTemplateDownloaderImpl implements DirectTemplateDown
private String downloadedFilePath;
private String installPath;
private String checksum;
public static final Logger s_logger = Logger.getLogger(DirectTemplateDownloaderImpl.class.getName());
protected DirectTemplateDownloaderImpl(final String url, final String destPoolPath, final Long templateId, final String checksum) {
this.url = url;
@ -70,6 +72,10 @@ public abstract class DirectTemplateDownloaderImpl implements DirectTemplateDown
return url;
}
public void setUrl(String url) {
this.url = url;
}
public String getDestPoolPath() {
return destPoolPath;
}
@ -152,11 +158,40 @@ public abstract class DirectTemplateDownloaderImpl implements DirectTemplateDown
return new DirectTemplateInformation(installPath, size, checksum);
}
/**
* Delete and create download file
*/
private void resetDownloadFile() {
File f = new File(getDownloadedFilePath());
s_logger.debug("Resetting download file: " + getDownloadedFilePath() + ", in order to re-download and persist template " + templateId + " on it");
try {
if (f.exists()) {
f.delete();
}
f.createNewFile();
} catch (IOException e) {
s_logger.error("Error creating file to download on: " + getDownloadedFilePath() + " due to: " + e.getMessage());
throw new CloudRuntimeException("Failed to create download file for direct download");
}
}
@Override
public boolean validateChecksum() {
if (StringUtils.isNotBlank(checksum)) {
int retry = 3;
boolean valid = false;
try {
return DigestHelper.check(checksum, new FileInputStream(downloadedFilePath));
while (!valid && retry > 0) {
s_logger.debug("Performing checksum validation for downloaded template " + templateId + ", retries left: " + retry);
valid = DigestHelper.check(checksum, new FileInputStream(downloadedFilePath));
retry--;
if (!valid && retry > 0) {
s_logger.debug("Checksum validation failded, re-downloading template");
resetDownloadFile();
downloadTemplate();
}
}
return valid;
} catch (IOException e) {
throw new CloudRuntimeException("could not check sum for file: " + downloadedFilePath);
} catch (NoSuchAlgorithmException e) {

6
agent/src/com/cloud/agent/direct/download/HttpDirectTemplateDownloader.java
View File

@ -36,16 +36,17 @@ import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.util.HashMap;
import java.util.Map;
public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
private HttpClient client;
protected HttpClient client;
private static final MultiThreadedHttpConnectionManager s_httpClientManager = new MultiThreadedHttpConnectionManager();
private static final int CHUNK_SIZE = 1024 * 1024; //1M
protected HttpMethodRetryHandler myretryhandler;
public static final Logger s_logger = Logger.getLogger(HttpDirectTemplateDownloader.class.getName());
protected GetMethod request;
protected Map<String, String> reqHeaders = new HashMap<>();
public HttpDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum, Map<String, String> headers) {
super(url, destPoolPath, templateId, checksum);
@ -69,6 +70,7 @@ public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
if (MapUtils.isNotEmpty(headers)) {
for (String key : headers.keySet()) {
request.setRequestHeader(key, headers.get(key));
reqHeaders.put(key, headers.get(key));
}
}
return request;

17
agent/src/com/cloud/agent/direct/download/HttpsDirectTemplateDownloader.java
View File

@ -21,6 +21,7 @@ package com.cloud.agent.direct.download;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.script.Script;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.io.IOUtils;
import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
@ -44,14 +45,15 @@ import java.security.KeyStore;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
import java.security.cert.CertificateException;
import java.util.Map;
public class HttpsDirectTemplateDownloader extends HttpDirectTemplateDownloader {
private CloseableHttpClient httpsClient;
private HttpUriRequest req;
public HttpsDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum) {
super(url, templateId, destPoolPath, checksum, null);
public HttpsDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum, Map<String, String> headers) {
super(url, templateId, destPoolPath, checksum, headers);
SSLContext sslcontext = null;
try {
sslcontext = getSSLContext();
@ -60,11 +62,16 @@ public class HttpsDirectTemplateDownloader extends HttpDirectTemplateDownloader
}
SSLConnectionSocketFactory factory = new SSLConnectionSocketFactory(sslcontext, SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
httpsClient = HttpClients.custom().setSSLSocketFactory(factory).build();
req = createUriRequest(url);
createUriRequest(url, headers);
}
protected HttpUriRequest createUriRequest(String downloadUrl) {
return new HttpGet(downloadUrl);
protected void createUriRequest(String downloadUrl, Map<String, String> headers) {
req = new HttpGet(downloadUrl);
if (MapUtils.isNotEmpty(headers)) {
for (String headerKey: headers.keySet()) {
req.setHeader(headerKey, headers.get(headerKey));
}
}
}
private SSLContext getSSLContext() throws KeyStoreException, NoSuchAlgorithmException, CertificateException, IOException, KeyManagementException {

52
agent/src/com/cloud/agent/direct/download/MetalinkDirectTemplateDownloader.java
View File

@ -18,32 +18,50 @@
//
package com.cloud.agent.direct.download;
import com.cloud.utils.script.Script;
import com.cloud.utils.UriUtils;
import org.apache.commons.collections.CollectionUtils;
import java.io.File;
import java.util.List;
import java.util.Map;
public class MetalinkDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
public class MetalinkDirectTemplateDownloader extends HttpDirectTemplateDownloader {
private String downloadDir;
public MetalinkDirectTemplateDownloader(String url, String destPoolPath, Long templateId, String checksum) {
super(url, destPoolPath, templateId, checksum);
String relativeDir = getDirectDownloadTempPath(templateId);
downloadDir = getDestPoolPath() + File.separator + relativeDir;
createFolder(downloadDir);
public MetalinkDirectTemplateDownloader(String url, String destPoolPath, Long templateId, String checksum, Map<String, String> headers) {
super(url, templateId, destPoolPath, checksum, headers);
}
@Override
public boolean downloadTemplate() {
String downloadCommand = "aria2c " + getUrl() + " -d " + downloadDir + " --check-integrity=true";
Script.runSimpleBashScript(downloadCommand);
//Remove .metalink file
Script.runSimpleBashScript("rm -f " + downloadDir + File.separator + getFileNameFromUrl());
String fileName = Script.runSimpleBashScript("ls " + downloadDir);
if (fileName == null) {
return false;
s_logger.debug("Retrieving metalink file from: " + getUrl() + " to file: " + getDownloadedFilePath());
List<String> metalinkUrls = UriUtils.getMetalinkUrls(getUrl());
if (CollectionUtils.isNotEmpty(metalinkUrls)) {
String downloadDir = getDirectDownloadTempPath(getTemplateId());
boolean downloaded = false;
int i = 0;
while (!downloaded && i < metalinkUrls.size()) {
try {
setUrl(metalinkUrls.get(i));
s_logger.debug("Trying to download template from metalink url: " + getUrl());
File f = new File(getDestPoolPath() + File.separator + downloadDir + File.separator + getFileNameFromUrl());
if (f.exists()) {
f.delete();
f.createNewFile();
}
setDownloadedFilePath(f.getAbsolutePath());
request = createRequest(getUrl(), reqHeaders);
downloaded = super.downloadTemplate();
if (downloaded) {
s_logger.debug("Successfully downloaded template from metalink url: " + getUrl());
break;
}
} catch (Exception e) {
s_logger.error("Error downloading template: " + getTemplateId() + " from " + getUrl() + ": " + e.getMessage());
}
i++;
}
return downloaded;
}
setDownloadedFilePath(downloadDir + File.separator + fileName);
return true;
}
}

10
core/src/org/apache/cloudstack/agent/directdownload/DirectDownloadCommand.java
View File

@ -23,6 +23,8 @@ import com.cloud.agent.api.Command;
import org.apache.cloudstack.storage.command.StorageSubSystemCommand;
import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
import java.util.Map;
public abstract class DirectDownloadCommand extends Command implements StorageSubSystemCommand {
public enum DownloadProtocol {
@ -33,12 +35,14 @@ public abstract class DirectDownloadCommand extends Command implements StorageSu
private Long templateId;
private PrimaryDataStoreTO destPool;
private String checksum;
private Map<String, String> headers;
protected DirectDownloadCommand (final String url, final Long templateId, final PrimaryDataStoreTO destPool, final String checksum) {
protected DirectDownloadCommand (final String url, final Long templateId, final PrimaryDataStoreTO destPool, final String checksum, final Map<String, String> headers) {
this.url = url;
this.templateId = templateId;
this.destPool = destPool;
this.checksum = checksum;
this.headers = headers;
}
public String getUrl() {
@ -57,6 +61,10 @@ public abstract class DirectDownloadCommand extends Command implements StorageSu
return checksum;
}
public Map<String, String> getHeaders() {
return headers;
}
@Override
public void setExecuteInSequence(boolean inSeq) {
}

9
core/src/org/apache/cloudstack/agent/directdownload/HttpDirectDownloadCommand.java
View File

@ -25,15 +25,8 @@ import java.util.Map;
public class HttpDirectDownloadCommand extends DirectDownloadCommand {
private Map<String, String> headers;
public HttpDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum, Map<String, String> headers) {
super(url, templateId, destPool, checksum);
this.headers = headers;
}
public Map<String, String> getHeaders() {
return headers;
super(url, templateId, destPool, checksum, headers);
}
}

2
core/src/org/apache/cloudstack/agent/directdownload/HttpsDirectDownloadCommand.java
View File

@ -26,6 +26,6 @@ import java.util.Map;
public class HttpsDirectDownloadCommand extends DirectDownloadCommand {
public HttpsDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum, Map<String, String> headers) {
super(url, templateId, destPool, checksum);
super(url, templateId, destPool, checksum, headers);
}
}

6
core/src/org/apache/cloudstack/agent/directdownload/MetalinkDirectDownloadCommand.java
View File

@ -21,10 +21,12 @@ package org.apache.cloudstack.agent.directdownload;
import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
import java.util.Map;
public class MetalinkDirectDownloadCommand extends DirectDownloadCommand {
public MetalinkDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum) {
super(url, templateId, destPool, checksum);
public MetalinkDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum, Map<String, String> headers) {
super(url, templateId, destPool, checksum, headers);
}
}

6
core/src/org/apache/cloudstack/agent/directdownload/NfsDirectDownloadCommand.java
View File

@ -21,10 +21,12 @@ package org.apache.cloudstack.agent.directdownload;
import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
import java.util.Map;
public class NfsDirectDownloadCommand extends DirectDownloadCommand {
public NfsDirectDownloadCommand(final String url, final Long templateId, final PrimaryDataStoreTO destPool, final String checksum) {
super(url, templateId, destPool, checksum);
public NfsDirectDownloadCommand(final String url, final Long templateId, final PrimaryDataStoreTO destPool, final String checksum, final Map<String, String> headers) {
super(url, templateId, destPool, checksum, headers);
}
}

3
engine/schema/src/com/cloud/host/dao/HostDao.java
View File

@ -23,6 +23,7 @@ import com.cloud.host.Host;
import com.cloud.host.Host.Type;
import com.cloud.host.HostVO;
import com.cloud.host.Status;
import com.cloud.hypervisor.Hypervisor;
import com.cloud.info.RunningHostCountInfo;
import com.cloud.resource.ResourceState;
import com.cloud.utils.db.GenericDao;
@ -96,4 +97,6 @@ public interface HostDao extends GenericDao<HostVO, Long>, StateDao<Status, Stat
HostVO findByPublicIp(String publicIp);
HostVO findByIp(String publicIp);
List<HostVO> listByDataCenterIdAndHypervisorType(long zoneId, Hypervisor.HypervisorType hypervisorType);
}

33
engine/schema/src/com/cloud/host/dao/HostDaoImpl.java
View File

@ -31,6 +31,8 @@ import javax.ejb.Local;
import javax.inject.Inject;
import javax.persistence.TableGenerator;
import com.cloud.hypervisor.Hypervisor;
import com.cloud.org.Grouping;
import org.apache.log4j.Logger;
import org.springframework.stereotype.Component;
@ -427,6 +429,37 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
return listBy(sc);
}
@Override
public List<HostVO> listByDataCenterIdAndHypervisorType(long zoneId, Hypervisor.HypervisorType hypervisorType) {
SearchBuilder<ClusterVO> clusterSearch = _clusterDao.createSearchBuilder();
clusterSearch.and("allocationState", clusterSearch.entity().getAllocationState(), SearchCriteria.Op.EQ);
clusterSearch.and("hypervisorType", clusterSearch.entity().getHypervisorType(), SearchCriteria.Op.EQ);
SearchBuilder<HostVO> hostSearch = createSearchBuilder();
hostSearch.and("dc", hostSearch.entity().getDataCenterId(), SearchCriteria.Op.EQ);
hostSearch.and("type", hostSearch.entity().getType(), Op.EQ);
hostSearch.and("status", hostSearch.entity().getStatus(), Op.EQ);
hostSearch.and("resourceState", hostSearch.entity().getResourceState(), Op.EQ);
hostSearch.join("clusterSearch", clusterSearch, hostSearch.entity().getClusterId(), clusterSearch.entity().getId(), JoinBuilder.JoinType.INNER);
hostSearch.done();
SearchCriteria<HostVO> sc = hostSearch.create();
sc.setParameters("dc", zoneId);
sc.setParameters("type", Host.Type.Routing);
sc.setParameters("status", Status.Up);
sc.setParameters("resourceState", ResourceState.Enabled);
sc.setJoinParameters("clusterSearch", "allocationState", Grouping.AllocationState.Enabled);
sc.setJoinParameters("clusterSearch", "hypervisorType", hypervisorType.toString());
return listBy(sc);
}
@Override
public HostVO findByGuid(String guid) {
SearchCriteria<HostVO> sc = GuidSearch.create("guid", guid);

6
plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java
View File

@ -1312,13 +1312,13 @@ public class KVMStorageProcessor implements StorageProcessor {
DirectTemplateDownloader downloader;
if (cmd instanceof HttpDirectDownloadCommand) {
downloader = new HttpDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPool.getLocalPath(), cmd.getChecksum(), ((HttpDirectDownloadCommand) cmd).getHeaders());
downloader = new HttpDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPool.getLocalPath(), cmd.getChecksum(), cmd.getHeaders());
} else if (cmd instanceof HttpsDirectDownloadCommand) {
downloader = new HttpsDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPool.getLocalPath(), cmd.getChecksum());
downloader = new HttpsDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPool.getLocalPath(), cmd.getChecksum(), cmd.getHeaders());
} else if (cmd instanceof NfsDirectDownloadCommand) {
downloader = new NfsDirectTemplateDownloader(cmd.getUrl(), destPool.getLocalPath(), cmd.getTemplateId(), cmd.getChecksum());
} else if (cmd instanceof MetalinkDirectDownloadCommand) {
downloader = new MetalinkDirectTemplateDownloader(cmd.getUrl(), destPool.getLocalPath(), cmd.getTemplateId(), cmd.getChecksum());
downloader = new MetalinkDirectTemplateDownloader(cmd.getUrl(), destPool.getLocalPath(), cmd.getTemplateId(), cmd.getChecksum(), cmd.getHeaders());
} else {
return new DirectDownloadAnswer(false, "Unsupported protocol, please provide HTTP(S), NFS or a metalink");
}

75
server/src/org/apache/cloudstack/direct/download/DirectDownloadManagerImpl.java
View File

@ -24,7 +24,7 @@ import com.cloud.host.Host;
import com.cloud.host.HostVO;
import com.cloud.host.Status;
import com.cloud.host.dao.HostDao;
import com.cloud.hypervisor.Hypervisor;
import com.cloud.hypervisor.Hypervisor.HypervisorType;
import com.cloud.storage.DataStoreRole;
import com.cloud.storage.VMTemplateStoragePoolVO;
import com.cloud.storage.VMTemplateStorageResourceAssoc;
@ -40,6 +40,8 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Arrays;
import java.util.Collections;
import javax.inject.Inject;
import org.apache.cloudstack.agent.directdownload.HttpsDirectDownloadCommand;
@ -57,6 +59,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStore;
import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.log4j.Logger;
@ -136,6 +139,50 @@ public class DirectDownloadManagerImpl extends ManagerBase implements DirectDown
return headers;
}
/**
* Get running host IDs within the same hypervisor, cluster and datacenter than hostId. ID hostId is not included on the returned list
*/
protected List<Long> getRunningHostIdsInTheSameCluster(Long clusterId, long dataCenterId, HypervisorType hypervisorType, long hostId) {
List<HostVO> hosts = hostDao.listByDataCenterIdAndHypervisorType(dataCenterId, hypervisorType);
List<Long> list = new ArrayList<>();
if (CollectionUtils.isNotEmpty(list)) {
for (HostVO host : hosts) {
if (host.getHypervisorType().equals(hypervisorType) && host.getStatus().equals(Status.Up) &&
host.getType().equals(Host.Type.Routing) && host.getClusterId().equals(clusterId) &&
host.getId() != hostId) {
list.add(host.getId());
}
}
}
Collections.shuffle(list);
return list;
}
/**
* Create host IDs array having hostId as the first element
*/
protected Long[] createHostIdsList(List<Long> hostIds, long hostId) {
if (CollectionUtils.isEmpty(hostIds)) {
return Arrays.asList(hostId).toArray(new Long[1]);
}
Long[] ids = new Long[hostIds.size() + 1];
ids[0] = hostId;
int i = 1;
for (Long id : hostIds) {
ids[i] = id;
i++;
}
return ids;
}
/**
* Get hosts to retry download having hostId as the first element
*/
protected Long[] getHostsToRetryOn(Long clusterId, long dataCenterId, HypervisorType hypervisorType, long hostId) {
List<Long> hostIds = getRunningHostIdsInTheSameCluster(clusterId, dataCenterId, hypervisorType, hostId);
return createHostIdsList(hostIds, hostId);
}
@Override
public void downloadTemplate(long templateId, long poolId, long hostId) {
VMTemplateVO template = vmTemplateDao.findById(templateId);
@ -166,10 +213,22 @@ public class DirectDownloadManagerImpl extends ManagerBase implements DirectDown
DownloadProtocol protocol = getProtocolFromUrl(url);
DirectDownloadCommand cmd = getDirectDownloadCommandFromProtocol(protocol, url, templateId, to, checksum, headers);
Answer answer = agentManager.easySend(hostId, cmd);
if (answer == null || !answer.getResult()) {
throw new CloudRuntimeException("Host " + hostId + " could not download template " +
templateId + " on pool " + poolId);
boolean downloaded = false;
int retry = 3;
Long[] hostsToRetry = getHostsToRetryOn(host.getClusterId(), host.getDataCenterId(), host.getHypervisorType(), hostId);
int hostIndex = 0;
Answer answer = null;
Long hostToSendDownloadCmd = hostsToRetry[hostIndex];
while (!downloaded && retry > 0) {
s_logger.debug("Sending Direct download command to host " + hostToSendDownloadCmd);
answer = agentManager.easySend(hostToSendDownloadCmd, cmd);
downloaded = answer != null && answer.getResult();
hostToSendDownloadCmd = hostsToRetry[(hostIndex + 1) % hostsToRetry.length];
retry --;
}
if (!downloaded) {
throw new CloudRuntimeException("Template " + templateId + " could not be downloaded on pool " + poolId + ", failing after trying on several hosts");
}
VMTemplateStoragePoolVO sPoolRef = vmTemplatePoolDao.findByPoolTemplate(poolId, templateId);
@ -204,9 +263,9 @@ public class DirectDownloadManagerImpl extends ManagerBase implements DirectDown
} else if (protocol.equals(DownloadProtocol.HTTPS)) {
return new HttpsDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders);
} else if (protocol.equals(DownloadProtocol.NFS)) {
return new NfsDirectDownloadCommand(url, templateId, destPool, checksum);
return new NfsDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders);
} else if (protocol.equals(DownloadProtocol.METALINK)) {
return new MetalinkDirectDownloadCommand(url, templateId, destPool, checksum);
return new MetalinkDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders);
} else {
return null;
}
@ -218,7 +277,7 @@ public class DirectDownloadManagerImpl extends ManagerBase implements DirectDown
List<HostVO> hosts = new ArrayList<>();
for (HostVO host: hostsQuery) {
if (host.getStatus().equals(Status.Up) &&
host.getHypervisorType().equals(Hypervisor.HypervisorType.KVM)) {
host.getHypervisorType().equals(HypervisorType.KVM)) {
hosts.add(host);
}
}

71
utils/src/com/cloud/utils/UriUtils.java
View File

@ -19,14 +19,22 @@
package com.cloud.utils;
import java.util.ArrayList;
import java.util.List;
import java.util.ListIterator;
import java.util.StringTokenizer;
import java.util.Collections;
import java.util.Comparator;
import java.util.Map;
import java.util.HashMap;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.w3c.dom.NamedNodeMap;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
@ -37,10 +45,6 @@ import java.net.URI;
import java.net.URISyntaxException;
import java.net.URLEncoder;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.List;
import java.util.ListIterator;
import java.util.StringTokenizer;
import javax.net.ssl.HttpsURLConnection;
@ -289,6 +293,37 @@ public class UriUtils {
}
}
/**
* Get node priority value (if provided), MAX_VALUE if not
*/
protected static Integer getNodePriority(Node node) {
if (node.hasAttributes()) {
NamedNodeMap attributes = node.getAttributes();
for (int k=0; k<attributes.getLength(); k++) {
Node attr = attributes.item(k);
if (attr.getNodeName().equals("priority")) {
String prio = attr.getNodeValue().replace("\"", "");
return Integer.valueOf(prio);
}
}
}
return Integer.MAX_VALUE;
}
/**
* Return the list of first elements on the list of pairs
*/
protected static List<String> getListOfFirstElements(List<Pair<String, Integer>> priorityList) {
List < String > values = new ArrayList<>();
for (Pair<String, Integer> pair : priorityList) {
values.add(pair.first());
}
return values;
}
/**
* Retrieve values from XML documents ordered by ascending priority for each tag name
*/
protected static Map<String, List<String>> getMultipleValuesFromXML(InputStream is, String[] tagNames) {
Map<String, List<String>> returnValues = new HashMap<String, List<String>>();
try {
@ -299,14 +334,21 @@ public class UriUtils {
for (int i = 0; i < tagNames.length; i++) {
NodeList targetNodes = rootElement.getElementsByTagName(tagNames[i]);
if (targetNodes.getLength() <= 0) {
s_logger.error("no " + tagNames[i] + " tag in XML response...returning null");
s_logger.error("no " + tagNames[i] + " tag in XML response...");
} else {
List<String> valueList = new ArrayList<String>();
List<Pair<String, Integer>> priorityList = new ArrayList<>();
for (int j = 0; j < targetNodes.getLength(); j++) {
Node node = targetNodes.item(j);
valueList.add(node.getTextContent());
Integer priority = getNodePriority(node);
priorityList.add(new Pair<>(node.getTextContent(), priority));
}
returnValues.put(tagNames[i], valueList);
Collections.sort(priorityList, new Comparator<Pair<String, Integer>>() {
@Override
public int compare(Pair<String, Integer> x, Pair<String, Integer> y) {
return x.second().compareTo(y.second());
}
});
returnValues.put(tagNames[i], getListOfFirstElements(priorityList));
}
}
} catch (Exception ex) {
@ -353,18 +395,23 @@ public class UriUtils {
}
/**
* Get list of urls on metalink
* @param metalinkUrl
* @return
* Get list of urls on metalink ordered by ascending priority (for those which priority tag is not defined, highest priority value is assumed)
*/
public static List<String> getMetalinkUrls(String metalinkUrl) {
HttpClient httpClient = new HttpClient(new MultiThreadedHttpConnectionManager());
GetMethod getMethod = new GetMethod(metalinkUrl);
List<String> urls = new ArrayList<>();
int status;
try {
status = httpClient.executeMethod(getMethod);
} catch (IOException e) {
s_logger.error("Error retrieving urls form metalink: " + metalinkUrl);
return null;
}
try (
InputStream is = getMethod.getResponseBodyAsStream()
) {
if (httpClient.executeMethod(getMethod) == HttpStatus.SC_OK) {
if (status == HttpStatus.SC_OK) {
Map<String, List<String>> metalinkUrlsMap = getMultipleValuesFromXML(is, new String[] {"url"});
if (metalinkUrlsMap.containsKey("url")) {
List<String> metalinkUrls = metalinkUrlsMap.get("url");