From 0ad2370baf4697932300d821824540e20807ad3e Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 26 Feb 2020 18:08:31 +0530 Subject: [PATCH] Enable Direct Download for System VMs (#3731) * changes for configurable timeouts for direct download Signed-off-by: Abhishek Kumar * server: refactor direct download config value retrieval Signed-off-by: Abhishek Kumar * refactored direc download cmd, downloader classes Signed-off-by: Abhishek Kumar * server, services: allow direct download template for SSVM, CPVM Signed-off-by: Abhishek Kumar * list bypassed system templates Signed-off-by: Abhishek Kumar * fix Signed-off-by: Abhishek Kumar * ignore direct download template during system tempalte download Signed-off-by: Abhishek Kumar * add direct download entry while adding store Signed-off-by: Abhishek Kumar * fix previous change, donot add multiple entries for direct download Signed-off-by: Abhishek Kumar * connection request timeout as hidden configuration Signed-off-by: Abhishek Kumar * fix template zone ref cleanup on zone deletion Signed-off-by: Abhishek Kumar * fix previous commit test error, change implementation Signed-off-by: Abhishek Kumar * refactored zone template cleanup Signed-off-by: Abhishek Kumar --- .../HttpDirectTemplateDownloader.java | 25 +-- .../HttpsDirectTemplateDownloader.java | 46 ++--- .../MetalinkDirectTemplateDownloader.java | 17 +- .../download/DirectDownloadManager.java | 25 ++- .../directdownload/DirectDownloadCommand.java | 36 +++- .../HttpDirectDownloadCommand.java | 8 +- .../HttpsDirectDownloadCommand.java | 8 +- .../MetalinkDirectDownloadCommand.java | 8 +- .../NfsDirectDownloadCommand.java | 6 +- .../cloud/storage/dao/VMTemplateDaoImpl.java | 4 +- .../cloud/storage/dao/VMTemplateZoneDao.java | 11 +- .../storage/dao/VMTemplateZoneDaoImpl.java | 9 + .../storage/image/TemplateServiceImpl.java | 7 +- .../kvm/storage/KVMStorageProcessor.java | 6 +- .../ConfigurationManagerImpl.java | 185 +++++++++--------- .../consoleproxy/ConsoleProxyManagerImpl.java | 8 +- .../download/DirectDownloadManagerImpl.java | 109 ++++++----- .../test/resources/createNetworkOffering.xml | 1 + .../SecondaryStorageManagerImpl.java | 3 +- 19 files changed, 307 insertions(+), 215 deletions(-) diff --git a/agent/src/main/java/com/cloud/agent/direct/download/HttpDirectTemplateDownloader.java b/agent/src/main/java/com/cloud/agent/direct/download/HttpDirectTemplateDownloader.java index 147ccabf8fc..f91af40dd40 100644 --- a/agent/src/main/java/com/cloud/agent/direct/download/HttpDirectTemplateDownloader.java +++ b/agent/src/main/java/com/cloud/agent/direct/download/HttpDirectTemplateDownloader.java @@ -19,22 +19,23 @@ package com.cloud.agent.direct.download; -import com.cloud.utils.exception.CloudRuntimeException; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.HashMap; +import java.util.Map; + import org.apache.commons.collections.MapUtils; import org.apache.commons.httpclient.HttpClient; -import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager; import org.apache.commons.httpclient.HttpStatus; +import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager; import org.apache.commons.httpclient.methods.GetMethod; import org.apache.commons.io.IOUtils; import org.apache.log4j.Logger; -import java.io.File; -import java.io.FileOutputStream; -import java.io.InputStream; -import java.io.OutputStream; -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; +import com.cloud.utils.exception.CloudRuntimeException; public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl { @@ -44,10 +45,10 @@ public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl { protected GetMethod request; protected Map reqHeaders = new HashMap<>(); - public HttpDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum, Map headers) { + public HttpDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum, Map headers, Integer connectTimeout, Integer soTimeout) { super(url, destPoolPath, templateId, checksum); - s_httpClientManager.getParams().setConnectionTimeout(5000); - s_httpClientManager.getParams().setSoTimeout(5000); + s_httpClientManager.getParams().setConnectionTimeout(connectTimeout == null ? 5000 : connectTimeout); + s_httpClientManager.getParams().setSoTimeout(soTimeout == null ? 5000 : soTimeout); client = new HttpClient(s_httpClientManager); request = createRequest(url, headers); String downloadDir = getDirectDownloadTempPath(templateId); diff --git a/agent/src/main/java/com/cloud/agent/direct/download/HttpsDirectTemplateDownloader.java b/agent/src/main/java/com/cloud/agent/direct/download/HttpsDirectTemplateDownloader.java index 38f59837cd8..f320846e75d 100644 --- a/agent/src/main/java/com/cloud/agent/direct/download/HttpsDirectTemplateDownloader.java +++ b/agent/src/main/java/com/cloud/agent/direct/download/HttpsDirectTemplateDownloader.java @@ -19,13 +19,26 @@ package com.cloud.agent.direct.download; -import com.cloud.utils.exception.CloudRuntimeException; -import com.cloud.utils.script.Script; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.security.KeyManagementException; +import java.security.KeyStore; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.security.cert.CertificateException; +import java.util.Map; + +import javax.net.ssl.SSLContext; + +import org.apache.commons.collections.MapUtils; import org.apache.commons.io.IOUtils; import org.apache.http.HttpEntity; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.commons.collections.MapUtils; import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpUriRequest; import org.apache.http.conn.ssl.SSLConnectionSocketFactory; @@ -34,27 +47,16 @@ import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; import org.apache.http.ssl.SSLContexts; -import javax.net.ssl.SSLContext; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.io.FileOutputStream; -import java.security.KeyManagementException; -import java.security.KeyStore; -import java.security.KeyStoreException; -import java.security.NoSuchAlgorithmException; -import java.security.cert.CertificateException; -import java.util.Map; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.script.Script; public class HttpsDirectTemplateDownloader extends HttpDirectTemplateDownloader { private CloseableHttpClient httpsClient; private HttpUriRequest req; - public HttpsDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum, Map headers) { - super(url, templateId, destPoolPath, checksum, headers); + public HttpsDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum, Map headers, Integer connectTimeout, Integer soTimeout, Integer connectionRequestTimeout) { + super(url, templateId, destPoolPath, checksum, headers, connectTimeout, soTimeout); SSLContext sslcontext = null; try { sslcontext = getSSLContext(); @@ -63,9 +65,9 @@ public class HttpsDirectTemplateDownloader extends HttpDirectTemplateDownloader } SSLConnectionSocketFactory factory = new SSLConnectionSocketFactory(sslcontext, SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER); RequestConfig config = RequestConfig.custom() - .setConnectTimeout(5000) - .setConnectionRequestTimeout(5000) - .setSocketTimeout(5000).build(); + .setConnectTimeout(connectTimeout == null ? 5000 : connectTimeout) + .setConnectionRequestTimeout(connectionRequestTimeout == null ? 5000 : connectionRequestTimeout) + .setSocketTimeout(soTimeout == null ? 5000 : soTimeout).build(); httpsClient = HttpClients.custom().setSSLSocketFactory(factory).setDefaultRequestConfig(config).build(); createUriRequest(url, headers); } diff --git a/agent/src/main/java/com/cloud/agent/direct/download/MetalinkDirectTemplateDownloader.java b/agent/src/main/java/com/cloud/agent/direct/download/MetalinkDirectTemplateDownloader.java index 2fd8ba03611..0a571015bee 100644 --- a/agent/src/main/java/com/cloud/agent/direct/download/MetalinkDirectTemplateDownloader.java +++ b/agent/src/main/java/com/cloud/agent/direct/download/MetalinkDirectTemplateDownloader.java @@ -18,17 +18,18 @@ // package com.cloud.agent.direct.download; -import com.cloud.utils.UriUtils; -import com.cloud.utils.exception.CloudRuntimeException; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.log4j.Logger; - import java.io.File; import java.util.List; import java.util.Map; import java.util.Random; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.log4j.Logger; + +import com.cloud.utils.UriUtils; +import com.cloud.utils.exception.CloudRuntimeException; + public class MetalinkDirectTemplateDownloader extends HttpDirectTemplateDownloader { private String metalinkUrl; @@ -37,8 +38,8 @@ public class MetalinkDirectTemplateDownloader extends HttpDirectTemplateDownload private Random random = new Random(); private static final Logger s_logger = Logger.getLogger(MetalinkDirectTemplateDownloader.class.getName()); - public MetalinkDirectTemplateDownloader(String url, String destPoolPath, Long templateId, String checksum, Map headers) { - super(url, templateId, destPoolPath, checksum, headers); + public MetalinkDirectTemplateDownloader(String url, String destPoolPath, Long templateId, String checksum, Map headers, Integer connectTimeout, Integer soTimeout) { + super(url, templateId, destPoolPath, checksum, headers, connectTimeout, soTimeout); metalinkUrl = url; metalinkUrls = UriUtils.getMetalinkUrls(metalinkUrl); metalinkChecksums = UriUtils.getMetalinkChecksums(metalinkUrl); diff --git a/api/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadManager.java b/api/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadManager.java index d627ffa69d1..f7dfae132a8 100644 --- a/api/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadManager.java +++ b/api/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadManager.java @@ -17,13 +17,18 @@ package org.apache.cloudstack.direct.download; -import com.cloud.utils.component.PluggableService; import org.apache.cloudstack.framework.agent.direct.download.DirectDownloadService; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; +import com.cloud.utils.component.PluggableService; + public interface DirectDownloadManager extends DirectDownloadService, PluggableService, Configurable { + static final int DEFAULT_DIRECT_DOWNLOAD_CONNECT_TIMEOUT = 5000; + static final int DEFAULT_DIRECT_DOWNLOAD_SOCKET_TIMEOUT = 5000; + static final int DEFAULT_DIRECT_DOWNLOAD_CONNECTION_REQUEST_TIMEOUT = 5000; + ConfigKey DirectDownloadCertificateUploadInterval = new ConfigKey<>("Advanced", Long.class, "direct.download.certificate.background.task.interval", "0", @@ -32,6 +37,24 @@ public interface DirectDownloadManager extends DirectDownloadService, PluggableS "Only certificates which have not been revoked from hosts are uploaded", false); + static final ConfigKey DirectDownloadConnectTimeout = new ConfigKey("Advanced", Integer.class, + "direct.download.connect.timeout", + String.valueOf(DEFAULT_DIRECT_DOWNLOAD_CONNECT_TIMEOUT), + "Connection establishment timeout in milliseconds for direct download", + true); + + static final ConfigKey DirectDownloadSocketTimeout = new ConfigKey("Advanced", Integer.class, + "direct.download.socket.timeout", + String.valueOf(DEFAULT_DIRECT_DOWNLOAD_SOCKET_TIMEOUT), + "Socket timeout (SO_TIMEOUT) in milliseconds for direct download", + true); + + static final ConfigKey DirectDownloadConnectionRequestTimeout = new ConfigKey("Hidden", Integer.class, + "direct.download.connection.request.timeout", + String.valueOf(DEFAULT_DIRECT_DOWNLOAD_CONNECTION_REQUEST_TIMEOUT), + "Requesting a connection from connection manager timeout in milliseconds for direct download", + true); + /** * Revoke direct download certificate with alias 'alias' from hosts of hypervisor type 'hypervisor' */ diff --git a/core/src/main/java/org/apache/cloudstack/agent/directdownload/DirectDownloadCommand.java b/core/src/main/java/org/apache/cloudstack/agent/directdownload/DirectDownloadCommand.java index 7a05d6144e4..506cd3140de 100644 --- a/core/src/main/java/org/apache/cloudstack/agent/directdownload/DirectDownloadCommand.java +++ b/core/src/main/java/org/apache/cloudstack/agent/directdownload/DirectDownloadCommand.java @@ -19,11 +19,11 @@ package org.apache.cloudstack.agent.directdownload; +import java.util.Map; + import org.apache.cloudstack.storage.command.StorageSubSystemCommand; import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; -import java.util.Map; - public abstract class DirectDownloadCommand extends StorageSubSystemCommand { public enum DownloadProtocol { @@ -35,13 +35,19 @@ public abstract class DirectDownloadCommand extends StorageSubSystemCommand { private PrimaryDataStoreTO destPool; private String checksum; private Map headers; + private Integer connectTimeout; + private Integer soTimeout; + private Integer connectionRequestTimeout; - protected DirectDownloadCommand (final String url, final Long templateId, final PrimaryDataStoreTO destPool, final String checksum, final Map headers) { + protected DirectDownloadCommand (final String url, final Long templateId, final PrimaryDataStoreTO destPool, final String checksum, final Map headers, final Integer connectTimeout, final Integer soTimeout, final Integer connectionRequestTimeout) { this.url = url; this.templateId = templateId; this.destPool = destPool; this.checksum = checksum; this.headers = headers; + this.connectTimeout = connectTimeout; + this.soTimeout = soTimeout; + this.connectionRequestTimeout = connectionRequestTimeout; } public String getUrl() { @@ -64,6 +70,30 @@ public abstract class DirectDownloadCommand extends StorageSubSystemCommand { return headers; } + public Integer getConnectTimeout() { + return connectTimeout; + } + + public void setConnectTimeout(Integer connectTimeout) { + this.connectTimeout = connectTimeout; + } + + public Integer getSoTimeout() { + return soTimeout; + } + + public void setSoTimeout(Integer soTimeout) { + this.soTimeout = soTimeout; + } + + public Integer getConnectionRequestTimeout() { + return connectionRequestTimeout; + } + + public void setConnectionRequestTimeout(Integer connectionRequestTimeout) { + this.connectionRequestTimeout = connectionRequestTimeout; + } + @Override public void setExecuteInSequence(boolean inSeq) { } diff --git a/core/src/main/java/org/apache/cloudstack/agent/directdownload/HttpDirectDownloadCommand.java b/core/src/main/java/org/apache/cloudstack/agent/directdownload/HttpDirectDownloadCommand.java index 7e32688154c..f131b3b0a7f 100644 --- a/core/src/main/java/org/apache/cloudstack/agent/directdownload/HttpDirectDownloadCommand.java +++ b/core/src/main/java/org/apache/cloudstack/agent/directdownload/HttpDirectDownloadCommand.java @@ -18,14 +18,14 @@ */ package org.apache.cloudstack.agent.directdownload; -import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; - import java.util.Map; +import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; + public class HttpDirectDownloadCommand extends DirectDownloadCommand { - public HttpDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum, Map headers) { - super(url, templateId, destPool, checksum, headers); + public HttpDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum, Map headers, int connectTimeout, int soTimeout) { + super(url, templateId, destPool, checksum, headers, connectTimeout, soTimeout, null); } } diff --git a/core/src/main/java/org/apache/cloudstack/agent/directdownload/HttpsDirectDownloadCommand.java b/core/src/main/java/org/apache/cloudstack/agent/directdownload/HttpsDirectDownloadCommand.java index ca926f1c761..8c415b6bd8d 100644 --- a/core/src/main/java/org/apache/cloudstack/agent/directdownload/HttpsDirectDownloadCommand.java +++ b/core/src/main/java/org/apache/cloudstack/agent/directdownload/HttpsDirectDownloadCommand.java @@ -19,13 +19,13 @@ package org.apache.cloudstack.agent.directdownload; -import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; - import java.util.Map; +import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; + public class HttpsDirectDownloadCommand extends DirectDownloadCommand { - public HttpsDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum, Map headers) { - super(url, templateId, destPool, checksum, headers); + public HttpsDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum, Map headers, int connectTimeout, int soTimeout, int connectionRequestTimeout) { + super(url, templateId, destPool, checksum, headers, connectTimeout, soTimeout, connectionRequestTimeout); } } \ No newline at end of file diff --git a/core/src/main/java/org/apache/cloudstack/agent/directdownload/MetalinkDirectDownloadCommand.java b/core/src/main/java/org/apache/cloudstack/agent/directdownload/MetalinkDirectDownloadCommand.java index da528d96694..a3edcebe7de 100644 --- a/core/src/main/java/org/apache/cloudstack/agent/directdownload/MetalinkDirectDownloadCommand.java +++ b/core/src/main/java/org/apache/cloudstack/agent/directdownload/MetalinkDirectDownloadCommand.java @@ -18,14 +18,14 @@ // package org.apache.cloudstack.agent.directdownload; -import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; - import java.util.Map; +import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; + public class MetalinkDirectDownloadCommand extends DirectDownloadCommand { - public MetalinkDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum, Map headers) { - super(url, templateId, destPool, checksum, headers); + public MetalinkDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum, Map headers, int connectTimeout, int soTimeout) { + super(url, templateId, destPool, checksum, headers, connectTimeout, soTimeout, null); } } diff --git a/core/src/main/java/org/apache/cloudstack/agent/directdownload/NfsDirectDownloadCommand.java b/core/src/main/java/org/apache/cloudstack/agent/directdownload/NfsDirectDownloadCommand.java index abc0137dfbd..0bf9c4d934b 100644 --- a/core/src/main/java/org/apache/cloudstack/agent/directdownload/NfsDirectDownloadCommand.java +++ b/core/src/main/java/org/apache/cloudstack/agent/directdownload/NfsDirectDownloadCommand.java @@ -18,14 +18,14 @@ // package org.apache.cloudstack.agent.directdownload; -import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; - import java.util.Map; +import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; + public class NfsDirectDownloadCommand extends DirectDownloadCommand { public NfsDirectDownloadCommand(final String url, final Long templateId, final PrimaryDataStoreTO destPool, final String checksum, final Map headers) { - super(url, templateId, destPool, checksum, headers); + super(url, templateId, destPool, checksum, headers, null, null, null); } } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java index dd1f2fcf164..6773c205aa3 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java @@ -345,7 +345,7 @@ public class VMTemplateDaoImpl extends GenericDaoBase implem readySystemTemplateSearch.and("state", readySystemTemplateSearch.entity().getState(), SearchCriteria.Op.EQ); readySystemTemplateSearch.and("templateType", readySystemTemplateSearch.entity().getTemplateType(), SearchCriteria.Op.EQ); SearchBuilder templateDownloadSearch = _templateDataStoreDao.createSearchBuilder(); - templateDownloadSearch.and("downloadState", templateDownloadSearch.entity().getDownloadState(), SearchCriteria.Op.EQ); + templateDownloadSearch.and("downloadState", templateDownloadSearch.entity().getDownloadState(), SearchCriteria.Op.IN); readySystemTemplateSearch.join("vmTemplateJoinTemplateStoreRef", templateDownloadSearch, templateDownloadSearch.entity().getTemplateId(), readySystemTemplateSearch.entity().getId(), JoinBuilder.JoinType.INNER); SearchBuilder hostHyperSearch2 = _hostDao.createSearchBuilder(); @@ -860,7 +860,7 @@ public class VMTemplateDaoImpl extends GenericDaoBase implem sc.setParameters("state", VirtualMachineTemplate.State.Active); sc.setJoinParameters("tmplHyper", "type", Host.Type.Routing); sc.setJoinParameters("tmplHyper", "zoneId", zoneId); - sc.setJoinParameters("vmTemplateJoinTemplateStoreRef", "downloadState", VMTemplateStorageResourceAssoc.Status.DOWNLOADED); + sc.setJoinParameters("vmTemplateJoinTemplateStoreRef", "downloadState", new VMTemplateStorageResourceAssoc.Status[] {VMTemplateStorageResourceAssoc.Status.DOWNLOADED, VMTemplateStorageResourceAssoc.Status.BYPASSED}); // order by descending order of id List tmplts = listBy(sc, new Filter(VMTemplateVO.class, "id", false, null, null)); diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateZoneDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateZoneDao.java index 67f7c3f64d7..66f474a1d7f 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateZoneDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateZoneDao.java @@ -22,14 +22,15 @@ import com.cloud.storage.VMTemplateZoneVO; import com.cloud.utils.db.GenericDao; public interface VMTemplateZoneDao extends GenericDao { - public List listByZoneId(long id); + List listByZoneId(long id); - public List listByTemplateId(long templateId); + List listByTemplateId(long templateId); - public VMTemplateZoneVO findByZoneTemplate(long zoneId, long templateId); + VMTemplateZoneVO findByZoneTemplate(long zoneId, long templateId); - public List listByZoneTemplate(Long zoneId, long templateId); + List listByZoneTemplate(Long zoneId, long templateId); - public void deletePrimaryRecordsForTemplate(long templateId); + void deletePrimaryRecordsForTemplate(long templateId); + void deleteByZoneId(long zoneId); } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateZoneDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateZoneDaoImpl.java index 21b3150bfd9..489ac130eaf 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateZoneDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateZoneDaoImpl.java @@ -93,4 +93,13 @@ public class VMTemplateZoneDaoImpl extends GenericDaoBase sc = ZoneSearch.create(); + sc.setParameters("zone_id", zoneId); + TransactionLegacy txn = TransactionLegacy.currentTxn(); + txn.start(); + remove(sc); + txn.commit(); + } } diff --git a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java index 97ac7c9037d..edf824403e1 100644 --- a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java +++ b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java @@ -264,13 +264,13 @@ public class TemplateServiceImpl implements TemplateService { List defaultBuiltin = _templateDao.listDefaultBuiltinTemplates(); for (VMTemplateVO rtngTmplt : rtngTmplts) { - if (rtngTmplt.getHypervisorType() == hostHyper) { + if (rtngTmplt.getHypervisorType() == hostHyper && !rtngTmplt.isDirectDownload()) { toBeDownloaded.add(rtngTmplt); } } for (VMTemplateVO builtinTmplt : defaultBuiltin) { - if (builtinTmplt.getHypervisorType() == hostHyper) { + if (builtinTmplt.getHypervisorType() == hostHyper && !builtinTmplt.isDirectDownload()) { toBeDownloaded.add(builtinTmplt); } } @@ -1201,6 +1201,9 @@ public class TemplateServiceImpl implements TemplateService { for (VMTemplateVO tmplt : rtngTmplts) { TemplateDataStoreVO tmpltStore = _vmTemplateStoreDao.findByStoreTemplate(storeId, tmplt.getId()); if (tmpltStore == null) { + if (_vmTemplateStoreDao.isTemplateMarkedForDirectDownload(tmplt.getId())) { + continue; + } tmpltStore = new TemplateDataStoreVO(storeId, tmplt.getId(), new Date(), 100, Status.DOWNLOADED, null, null, null, TemplateConstants.DEFAULT_SYSTEM_VM_TEMPLATE_PATH + tmplt.getId() + '/', tmplt.getUrl()); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java index 9a2fd275dd3..979583b1399 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java @@ -1678,13 +1678,13 @@ public class KVMStorageProcessor implements StorageProcessor { */ private DirectTemplateDownloader getDirectTemplateDownloaderFromCommand(DirectDownloadCommand cmd, KVMStoragePool destPool) { if (cmd instanceof HttpDirectDownloadCommand) { - return new HttpDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPool.getLocalPath(), cmd.getChecksum(), cmd.getHeaders()); + return new HttpDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPool.getLocalPath(), cmd.getChecksum(), cmd.getHeaders(), cmd.getConnectTimeout(), cmd.getSoTimeout()); } else if (cmd instanceof HttpsDirectDownloadCommand) { - return new HttpsDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPool.getLocalPath(), cmd.getChecksum(), cmd.getHeaders()); + return new HttpsDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPool.getLocalPath(), cmd.getChecksum(), cmd.getHeaders(), cmd.getConnectTimeout(), cmd.getSoTimeout(), cmd.getConnectionRequestTimeout()); } else if (cmd instanceof NfsDirectDownloadCommand) { return new NfsDirectTemplateDownloader(cmd.getUrl(), destPool.getLocalPath(), cmd.getTemplateId(), cmd.getChecksum()); } else if (cmd instanceof MetalinkDirectDownloadCommand) { - return new MetalinkDirectTemplateDownloader(cmd.getUrl(), destPool.getLocalPath(), cmd.getTemplateId(), cmd.getChecksum(), cmd.getHeaders()); + return new MetalinkDirectTemplateDownloader(cmd.getUrl(), destPool.getLocalPath(), cmd.getTemplateId(), cmd.getChecksum(), cmd.getHeaders(), cmd.getConnectTimeout(), cmd.getSoTimeout()); } else { throw new IllegalArgumentException("Unsupported protocol, please provide HTTP(S), NFS or a metalink"); } diff --git a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java index be01b9f8325..05f5fffdb8a 100755 --- a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java +++ b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java @@ -16,6 +16,96 @@ // under the License. package com.cloud.configuration; +import java.io.UnsupportedEncodingException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URLDecoder; +import java.sql.Date; +import java.sql.PreparedStatement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.ListIterator; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.UUID; + +import javax.inject.Inject; +import javax.naming.ConfigurationException; + +import org.apache.cloudstack.acl.SecurityChecker; +import org.apache.cloudstack.affinity.AffinityGroup; +import org.apache.cloudstack.affinity.AffinityGroupService; +import org.apache.cloudstack.affinity.dao.AffinityGroupDao; +import org.apache.cloudstack.agent.lb.IndirectAgentLB; +import org.apache.cloudstack.agent.lb.IndirectAgentLBServiceImpl; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.command.admin.config.UpdateCfgCmd; +import org.apache.cloudstack.api.command.admin.network.CreateManagementNetworkIpRangeCmd; +import org.apache.cloudstack.api.command.admin.network.CreateNetworkOfferingCmd; +import org.apache.cloudstack.api.command.admin.network.DeleteManagementNetworkIpRangeCmd; +import org.apache.cloudstack.api.command.admin.network.DeleteNetworkOfferingCmd; +import org.apache.cloudstack.api.command.admin.network.UpdateNetworkOfferingCmd; +import org.apache.cloudstack.api.command.admin.offering.CreateDiskOfferingCmd; +import org.apache.cloudstack.api.command.admin.offering.CreateServiceOfferingCmd; +import org.apache.cloudstack.api.command.admin.offering.DeleteDiskOfferingCmd; +import org.apache.cloudstack.api.command.admin.offering.DeleteServiceOfferingCmd; +import org.apache.cloudstack.api.command.admin.offering.UpdateDiskOfferingCmd; +import org.apache.cloudstack.api.command.admin.offering.UpdateServiceOfferingCmd; +import org.apache.cloudstack.api.command.admin.pod.DeletePodCmd; +import org.apache.cloudstack.api.command.admin.pod.UpdatePodCmd; +import org.apache.cloudstack.api.command.admin.region.CreatePortableIpRangeCmd; +import org.apache.cloudstack.api.command.admin.region.DeletePortableIpRangeCmd; +import org.apache.cloudstack.api.command.admin.region.ListPortableIpRangesCmd; +import org.apache.cloudstack.api.command.admin.vlan.CreateVlanIpRangeCmd; +import org.apache.cloudstack.api.command.admin.vlan.DedicatePublicIpRangeCmd; +import org.apache.cloudstack.api.command.admin.vlan.DeleteVlanIpRangeCmd; +import org.apache.cloudstack.api.command.admin.vlan.ReleasePublicIpRangeCmd; +import org.apache.cloudstack.api.command.admin.zone.CreateZoneCmd; +import org.apache.cloudstack.api.command.admin.zone.DeleteZoneCmd; +import org.apache.cloudstack.api.command.admin.zone.UpdateZoneCmd; +import org.apache.cloudstack.api.command.user.network.ListNetworkOfferingsCmd; +import org.apache.cloudstack.config.ApiServiceConfiguration; +import org.apache.cloudstack.config.Configuration; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; +import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope; +import org.apache.cloudstack.framework.config.ConfigDepot; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.Configurable; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.framework.config.impl.ConfigurationVO; +import org.apache.cloudstack.framework.messagebus.MessageBus; +import org.apache.cloudstack.framework.messagebus.MessageSubscriber; +import org.apache.cloudstack.framework.messagebus.PublishScope; +import org.apache.cloudstack.region.PortableIp; +import org.apache.cloudstack.region.PortableIpDao; +import org.apache.cloudstack.region.PortableIpRange; +import org.apache.cloudstack.region.PortableIpRangeDao; +import org.apache.cloudstack.region.PortableIpRangeVO; +import org.apache.cloudstack.region.PortableIpVO; +import org.apache.cloudstack.region.Region; +import org.apache.cloudstack.region.RegionVO; +import org.apache.cloudstack.region.dao.RegionDao; +import org.apache.cloudstack.resourcedetail.DiskOfferingDetailVO; +import org.apache.cloudstack.resourcedetail.dao.DiskOfferingDetailsDao; +import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; +import org.apache.cloudstack.storage.datastore.db.ImageStoreDetailsDao; +import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.log4j.Logger; + import com.cloud.agent.AgentManager; import com.cloud.alert.AlertManager; import com.cloud.api.ApiDBUtils; @@ -118,6 +208,7 @@ import com.cloud.storage.DiskOfferingVO; import com.cloud.storage.Storage.ProvisioningType; import com.cloud.storage.StorageManager; import com.cloud.storage.dao.DiskOfferingDao; +import com.cloud.storage.dao.VMTemplateZoneDao; import com.cloud.storage.dao.VolumeDao; import com.cloud.test.IPRangeConfig; import com.cloud.user.Account; @@ -158,95 +249,6 @@ import com.google.common.base.MoreObjects; import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.collect.Sets; -import org.apache.cloudstack.acl.SecurityChecker; -import org.apache.cloudstack.affinity.AffinityGroup; -import org.apache.cloudstack.affinity.AffinityGroupService; -import org.apache.cloudstack.affinity.dao.AffinityGroupDao; -import org.apache.cloudstack.agent.lb.IndirectAgentLB; -import org.apache.cloudstack.agent.lb.IndirectAgentLBServiceImpl; -import org.apache.cloudstack.api.ApiConstants; -import org.apache.cloudstack.api.command.admin.config.UpdateCfgCmd; -import org.apache.cloudstack.api.command.admin.network.CreateManagementNetworkIpRangeCmd; -import org.apache.cloudstack.api.command.admin.network.CreateNetworkOfferingCmd; -import org.apache.cloudstack.api.command.admin.network.DeleteManagementNetworkIpRangeCmd; -import org.apache.cloudstack.api.command.admin.network.DeleteNetworkOfferingCmd; -import org.apache.cloudstack.api.command.admin.network.UpdateNetworkOfferingCmd; -import org.apache.cloudstack.api.command.admin.offering.CreateDiskOfferingCmd; -import org.apache.cloudstack.api.command.admin.offering.CreateServiceOfferingCmd; -import org.apache.cloudstack.api.command.admin.offering.DeleteDiskOfferingCmd; -import org.apache.cloudstack.api.command.admin.offering.DeleteServiceOfferingCmd; -import org.apache.cloudstack.api.command.admin.offering.UpdateDiskOfferingCmd; -import org.apache.cloudstack.api.command.admin.offering.UpdateServiceOfferingCmd; -import org.apache.cloudstack.api.command.admin.pod.DeletePodCmd; -import org.apache.cloudstack.api.command.admin.pod.UpdatePodCmd; -import org.apache.cloudstack.api.command.admin.region.CreatePortableIpRangeCmd; -import org.apache.cloudstack.api.command.admin.region.DeletePortableIpRangeCmd; -import org.apache.cloudstack.api.command.admin.region.ListPortableIpRangesCmd; -import org.apache.cloudstack.api.command.admin.vlan.CreateVlanIpRangeCmd; -import org.apache.cloudstack.api.command.admin.vlan.DedicatePublicIpRangeCmd; -import org.apache.cloudstack.api.command.admin.vlan.DeleteVlanIpRangeCmd; -import org.apache.cloudstack.api.command.admin.vlan.ReleasePublicIpRangeCmd; -import org.apache.cloudstack.api.command.admin.zone.CreateZoneCmd; -import org.apache.cloudstack.api.command.admin.zone.DeleteZoneCmd; -import org.apache.cloudstack.api.command.admin.zone.UpdateZoneCmd; -import org.apache.cloudstack.api.command.user.network.ListNetworkOfferingsCmd; -import org.apache.cloudstack.config.ApiServiceConfiguration; -import org.apache.cloudstack.config.Configuration; -import org.apache.cloudstack.context.CallContext; -import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; -import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; -import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope; -import org.apache.cloudstack.framework.config.ConfigDepot; -import org.apache.cloudstack.framework.config.ConfigKey; -import org.apache.cloudstack.framework.config.Configurable; -import org.apache.cloudstack.framework.config.dao.ConfigurationDao; -import org.apache.cloudstack.framework.config.impl.ConfigurationVO; -import org.apache.cloudstack.framework.messagebus.MessageBus; -import org.apache.cloudstack.framework.messagebus.MessageSubscriber; -import org.apache.cloudstack.framework.messagebus.PublishScope; -import org.apache.cloudstack.region.PortableIp; -import org.apache.cloudstack.region.PortableIpDao; -import org.apache.cloudstack.region.PortableIpRange; -import org.apache.cloudstack.region.PortableIpRangeDao; -import org.apache.cloudstack.region.PortableIpRangeVO; -import org.apache.cloudstack.region.PortableIpVO; -import org.apache.cloudstack.region.Region; -import org.apache.cloudstack.region.RegionVO; -import org.apache.cloudstack.region.dao.RegionDao; -import org.apache.cloudstack.resourcedetail.DiskOfferingDetailVO; -import org.apache.cloudstack.resourcedetail.dao.DiskOfferingDetailsDao; -import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; -import org.apache.cloudstack.storage.datastore.db.ImageStoreDetailsDao; -import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; -import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; -import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao; -import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.collections.MapUtils; -import org.apache.log4j.Logger; - -import java.io.UnsupportedEncodingException; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.URLDecoder; -import java.sql.Date; -import java.sql.PreparedStatement; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.ListIterator; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; -import java.util.UUID; - -import javax.inject.Inject; -import javax.naming.ConfigurationException; public class ConfigurationManagerImpl extends ManagerBase implements ConfigurationManager, ConfigurationService, Configurable { public static final Logger s_logger = Logger.getLogger(ConfigurationManagerImpl.class); @@ -381,7 +383,8 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati AgentManager _agentManager; @Inject IndirectAgentLB _indirectAgentLB; - + @Inject + private VMTemplateZoneDao templateZoneDao; // FIXME - why don't we have interface for DataCenterLinkLocalIpAddressDao? @Inject @@ -1838,6 +1841,8 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati final boolean success = _zoneDao.remove(zoneId); if (success) { + // delete template refs for this zone + templateZoneDao.deleteByZoneId(zoneId); // delete all capacity records for the zone _capacityDao.removeBy(null, zoneId, null, null, null); // remove from dedicated resources diff --git a/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java b/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java index a299937650c..368fc33876c 100644 --- a/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java +++ b/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java @@ -100,6 +100,7 @@ import com.cloud.resource.ServerResource; import com.cloud.resource.UnableDeleteHostException; import com.cloud.service.ServiceOfferingVO; import com.cloud.service.dao.ServiceOfferingDao; +import com.cloud.storage.DataStoreRole; import com.cloud.storage.Storage; import com.cloud.storage.StoragePoolStatus; import com.cloud.storage.VMTemplateStorageResourceAssoc.Status; @@ -1011,7 +1012,12 @@ public class ConsoleProxyManagerImpl extends ManagerBase implements ConsoleProxy } return false; } - TemplateDataStoreVO templateHostRef = _vmTemplateStoreDao.findByTemplateZoneDownloadStatus(template.getId(), dataCenterId, Status.DOWNLOADED); + TemplateDataStoreVO templateHostRef = null; + if (template.isDirectDownload()) { + templateHostRef = _vmTemplateStoreDao.findByTemplate(template.getId(), DataStoreRole.Image); + } else { + templateHostRef = _vmTemplateStoreDao.findByTemplateZoneDownloadStatus(template.getId(), dataCenterId, Status.DOWNLOADED); + } if (templateHostRef != null) { boolean useLocalStorage = false; diff --git a/server/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadManagerImpl.java b/server/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadManagerImpl.java index 2eb6d36b9bf..792a5a30ee8 100644 --- a/server/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadManagerImpl.java @@ -20,6 +20,53 @@ package org.apache.cloudstack.direct.download; import static com.cloud.storage.Storage.ImageFormat; +import java.net.URI; +import java.net.URISyntaxException; +import java.security.cert.Certificate; +import java.security.cert.CertificateException; +import java.security.cert.CertificateExpiredException; +import java.security.cert.CertificateNotYetValidException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import javax.inject.Inject; +import javax.naming.ConfigurationException; + +import org.apache.cloudstack.agent.directdownload.DirectDownloadAnswer; +import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand; +import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand.DownloadProtocol; +import org.apache.cloudstack.agent.directdownload.HttpDirectDownloadCommand; +import org.apache.cloudstack.agent.directdownload.HttpsDirectDownloadCommand; +import org.apache.cloudstack.agent.directdownload.MetalinkDirectDownloadCommand; +import org.apache.cloudstack.agent.directdownload.NfsDirectDownloadCommand; +import org.apache.cloudstack.agent.directdownload.RevokeDirectDownloadCertificateCommand; +import org.apache.cloudstack.agent.directdownload.SetupDirectDownloadCertificateCommand; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; +import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; +import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStore; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.managed.context.ManagedContextRunnable; +import org.apache.cloudstack.poll.BackgroundPollManager; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.log4j.Logger; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; + import com.cloud.agent.AgentManager; import com.cloud.agent.api.Answer; import com.cloud.dc.DataCenterVO; @@ -43,52 +90,8 @@ import com.cloud.storage.dao.VMTemplatePoolDao; import com.cloud.utils.component.ManagerBase; import com.cloud.utils.concurrency.NamedThreadFactory; import com.cloud.utils.exception.CloudRuntimeException; - -import java.net.URI; -import java.net.URISyntaxException; -import java.security.cert.Certificate; -import java.security.cert.CertificateException; -import java.security.cert.CertificateExpiredException; -import java.security.cert.CertificateNotYetValidException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Arrays; -import java.util.Collections; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; -import javax.inject.Inject; -import javax.naming.ConfigurationException; - import com.cloud.utils.security.CertificateHelper; -import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand; -import org.apache.cloudstack.agent.directdownload.DirectDownloadAnswer; -import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand.DownloadProtocol; -import org.apache.cloudstack.agent.directdownload.HttpDirectDownloadCommand; -import org.apache.cloudstack.agent.directdownload.MetalinkDirectDownloadCommand; -import org.apache.cloudstack.agent.directdownload.NfsDirectDownloadCommand; -import org.apache.cloudstack.agent.directdownload.HttpsDirectDownloadCommand; -import org.apache.cloudstack.agent.directdownload.RevokeDirectDownloadCertificateCommand; -import org.apache.cloudstack.agent.directdownload.SetupDirectDownloadCertificateCommand; -import org.apache.cloudstack.context.CallContext; -import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; -import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; -import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; -import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStore; -import org.apache.cloudstack.framework.config.ConfigKey; -import org.apache.cloudstack.managed.context.ManagedContextRunnable; -import org.apache.cloudstack.poll.BackgroundPollManager; -import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; -import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; -import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.collections.MapUtils; -import org.apache.log4j.Logger; -import org.joda.time.DateTime; -import org.joda.time.DateTimeZone; + import sun.security.x509.X509CertImpl; public class DirectDownloadManagerImpl extends ManagerBase implements DirectDownloadManager { @@ -119,6 +122,8 @@ public class DirectDownloadManagerImpl extends ManagerBase implements DirectDown private BackgroundPollManager backgroundPollManager; @Inject private DataCenterDao dataCenterDao; + @Inject + private ConfigurationDao configDao; protected ScheduledExecutorService executorService; @@ -320,14 +325,17 @@ public class DirectDownloadManagerImpl extends ManagerBase implements DirectDown */ private DirectDownloadCommand getDirectDownloadCommandFromProtocol(DownloadProtocol protocol, String url, Long templateId, PrimaryDataStoreTO destPool, String checksum, Map httpHeaders) { + int connectTimeout = DirectDownloadConnectTimeout.value(); + int soTimeout = DirectDownloadSocketTimeout.value(); + int connectionRequestTimeout = DirectDownloadConnectionRequestTimeout.value(); if (protocol.equals(DownloadProtocol.HTTP)) { - return new HttpDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders); + return new HttpDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders, connectTimeout, soTimeout); } else if (protocol.equals(DownloadProtocol.HTTPS)) { - return new HttpsDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders); + return new HttpsDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders, connectTimeout, soTimeout, connectionRequestTimeout); } else if (protocol.equals(DownloadProtocol.NFS)) { return new NfsDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders); } else if (protocol.equals(DownloadProtocol.METALINK)) { - return new MetalinkDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders); + return new MetalinkDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders, connectTimeout, soTimeout); } else { return null; } @@ -549,7 +557,10 @@ public class DirectDownloadManagerImpl extends ManagerBase implements DirectDown @Override public ConfigKey[] getConfigKeys() { return new ConfigKey[]{ - DirectDownloadCertificateUploadInterval + DirectDownloadCertificateUploadInterval, + DirectDownloadConnectTimeout, + DirectDownloadSocketTimeout, + DirectDownloadConnectionRequestTimeout }; } diff --git a/server/src/test/resources/createNetworkOffering.xml b/server/src/test/resources/createNetworkOffering.xml index 32596fc03a2..8dee0e8a54e 100644 --- a/server/src/test/resources/createNetworkOffering.xml +++ b/server/src/test/resources/createNetworkOffering.xml @@ -58,5 +58,6 @@ + diff --git a/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java b/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java index 8b2ed40c15d..a1a3873bf88 100644 --- a/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java +++ b/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java @@ -828,8 +828,7 @@ public class SecondaryStorageManagerImpl extends ManagerBase implements Secondar return false; } - DataStore store = templateMgr.getImageStore(dataCenterId, template.getId()); - if (store == null) { + if (!template.isDirectDownload() && templateMgr.getImageStore(dataCenterId, template.getId()) == null) { if (s_logger.isDebugEnabled()) { s_logger.debug("No secondary storage available in zone " + dataCenterId + ", wait until it is ready to launch secondary storage vm"); }