etan39
/
cloudstack
mirror of https://github.com/apache/cloudstack.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Enable Direct Download for System VMs (#3731) 

* changes for configurable timeouts for direct download

Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com>

* server: refactor direct download config value retrieval

Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com>

* refactored direc download cmd, downloader classes

Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com>

* server, services: allow direct download template for SSVM, CPVM

Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com>

* list bypassed system templates

Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com>

* fix

Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com>

* ignore direct download template during system tempalte download

Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com>

* add direct download entry while adding store

Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com>

* fix previous change, donot add multiple entries for direct download

Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com>

* connection request timeout as hidden configuration

Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com>

* fix template zone ref cleanup on zone deletion

Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com>

* fix previous commit test error, change implementation

Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com>

* refactored zone template cleanup

Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com>
This commit is contained in:
Abhishek Kumar 2020-02-26 18:08:31 +05:30 committed by GitHub
parent b96fbee9e5
commit 0ad2370baf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 307 additions and 215 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
agent/src/main/java/com/cloud/agent/direct/download/HttpDirectTemplateDownloader.java
View File

@ -19,22 +19,23 @@
package com.cloud.agent.direct.download;
import com.cloud.utils.exception.CloudRuntimeException;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.HashMap;
import java.util.Map;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.io.IOUtils;
import org.apache.log4j.Logger;
import java.io.File;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import com.cloud.utils.exception.CloudRuntimeException;
public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
@ -44,10 +45,10 @@ public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
protected GetMethod request;
protected Map<String, String> reqHeaders = new HashMap<>();
public HttpDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum, Map<String, String> headers) {
public HttpDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum, Map<String, String> headers, Integer connectTimeout, Integer soTimeout) {
super(url, destPoolPath, templateId, checksum);
s_httpClientManager.getParams().setConnectionTimeout(5000);
s_httpClientManager.getParams().setSoTimeout(5000);
s_httpClientManager.getParams().setConnectionTimeout(connectTimeout == null ? 5000 : connectTimeout);
s_httpClientManager.getParams().setSoTimeout(soTimeout == null ? 5000 : soTimeout);
client = new HttpClient(s_httpClientManager);
request = createRequest(url, headers);
String downloadDir = getDirectDownloadTempPath(templateId);

46
agent/src/main/java/com/cloud/agent/direct/download/HttpsDirectTemplateDownloader.java
View File

@ -19,13 +19,26 @@
package com.cloud.agent.direct.download;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.script.Script;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.security.KeyManagementException;
import java.security.KeyStore;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
import java.security.cert.CertificateException;
import java.util.Map;
import javax.net.ssl.SSLContext;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.io.IOUtils;
import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.commons.collections.MapUtils;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
@ -34,27 +47,16 @@ import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.ssl.SSLContexts;
import javax.net.ssl.SSLContext;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.FileOutputStream;
import java.security.KeyManagementException;
import java.security.KeyStore;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
import java.security.cert.CertificateException;
import java.util.Map;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.script.Script;
public class HttpsDirectTemplateDownloader extends HttpDirectTemplateDownloader {
private CloseableHttpClient httpsClient;
private HttpUriRequest req;
public HttpsDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum, Map<String, String> headers) {
super(url, templateId, destPoolPath, checksum, headers);
public HttpsDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum, Map<String, String> headers, Integer connectTimeout, Integer soTimeout, Integer connectionRequestTimeout) {
super(url, templateId, destPoolPath, checksum, headers, connectTimeout, soTimeout);
SSLContext sslcontext = null;
try {
sslcontext = getSSLContext();
@ -63,9 +65,9 @@ public class HttpsDirectTemplateDownloader extends HttpDirectTemplateDownloader
}
SSLConnectionSocketFactory factory = new SSLConnectionSocketFactory(sslcontext, SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
RequestConfig config = RequestConfig.custom()
.setConnectTimeout(5000)
.setConnectionRequestTimeout(5000)
.setSocketTimeout(5000).build();
.setConnectTimeout(connectTimeout == null ? 5000 : connectTimeout)
.setConnectionRequestTimeout(connectionRequestTimeout == null ? 5000 : connectionRequestTimeout)
.setSocketTimeout(soTimeout == null ? 5000 : soTimeout).build();
httpsClient = HttpClients.custom().setSSLSocketFactory(factory).setDefaultRequestConfig(config).build();
createUriRequest(url, headers);
}

17
agent/src/main/java/com/cloud/agent/direct/download/MetalinkDirectTemplateDownloader.java
View File

@ -18,17 +18,18 @@
//
package com.cloud.agent.direct.download;
import com.cloud.utils.UriUtils;
import com.cloud.utils.exception.CloudRuntimeException;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import java.io.File;
import java.util.List;
import java.util.Map;
import java.util.Random;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import com.cloud.utils.UriUtils;
import com.cloud.utils.exception.CloudRuntimeException;
public class MetalinkDirectTemplateDownloader extends HttpDirectTemplateDownloader {
private String metalinkUrl;
@ -37,8 +38,8 @@ public class MetalinkDirectTemplateDownloader extends HttpDirectTemplateDownload
private Random random = new Random();
private static final Logger s_logger = Logger.getLogger(MetalinkDirectTemplateDownloader.class.getName());
public MetalinkDirectTemplateDownloader(String url, String destPoolPath, Long templateId, String checksum, Map<String, String> headers) {
super(url, templateId, destPoolPath, checksum, headers);
public MetalinkDirectTemplateDownloader(String url, String destPoolPath, Long templateId, String checksum, Map<String, String> headers, Integer connectTimeout, Integer soTimeout) {
super(url, templateId, destPoolPath, checksum, headers, connectTimeout, soTimeout);
metalinkUrl = url;
metalinkUrls = UriUtils.getMetalinkUrls(metalinkUrl);
metalinkChecksums = UriUtils.getMetalinkChecksums(metalinkUrl);

25
api/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadManager.java
View File

@ -17,13 +17,18 @@
package org.apache.cloudstack.direct.download;
import com.cloud.utils.component.PluggableService;
import org.apache.cloudstack.framework.agent.direct.download.DirectDownloadService;
import org.apache.cloudstack.framework.config.ConfigKey;
import org.apache.cloudstack.framework.config.Configurable;
import com.cloud.utils.component.PluggableService;
public interface DirectDownloadManager extends DirectDownloadService, PluggableService, Configurable {
static final int DEFAULT_DIRECT_DOWNLOAD_CONNECT_TIMEOUT = 5000;
static final int DEFAULT_DIRECT_DOWNLOAD_SOCKET_TIMEOUT = 5000;
static final int DEFAULT_DIRECT_DOWNLOAD_CONNECTION_REQUEST_TIMEOUT = 5000;
ConfigKey<Long> DirectDownloadCertificateUploadInterval = new ConfigKey<>("Advanced", Long.class,
"direct.download.certificate.background.task.interval",
"0",
@ -32,6 +37,24 @@ public interface DirectDownloadManager extends DirectDownloadService, PluggableS
"Only certificates which have not been revoked from hosts are uploaded",
false);
static final ConfigKey<Integer> DirectDownloadConnectTimeout = new ConfigKey<Integer>("Advanced", Integer.class,
"direct.download.connect.timeout",
String.valueOf(DEFAULT_DIRECT_DOWNLOAD_CONNECT_TIMEOUT),
"Connection establishment timeout in milliseconds for direct download",
true);
static final ConfigKey<Integer> DirectDownloadSocketTimeout = new ConfigKey<Integer>("Advanced", Integer.class,
"direct.download.socket.timeout",
String.valueOf(DEFAULT_DIRECT_DOWNLOAD_SOCKET_TIMEOUT),
"Socket timeout (SO_TIMEOUT) in milliseconds for direct download",
true);
static final ConfigKey<Integer> DirectDownloadConnectionRequestTimeout = new ConfigKey<Integer>("Hidden", Integer.class,
"direct.download.connection.request.timeout",
String.valueOf(DEFAULT_DIRECT_DOWNLOAD_CONNECTION_REQUEST_TIMEOUT),
"Requesting a connection from connection manager timeout in milliseconds for direct download",
true);
/**
* Revoke direct download certificate with alias 'alias' from hosts of hypervisor type 'hypervisor'
*/

36
core/src/main/java/org/apache/cloudstack/agent/directdownload/DirectDownloadCommand.java
View File

@ -19,11 +19,11 @@
package org.apache.cloudstack.agent.directdownload;
import java.util.Map;
import org.apache.cloudstack.storage.command.StorageSubSystemCommand;
import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
import java.util.Map;
public abstract class DirectDownloadCommand extends StorageSubSystemCommand {
public enum DownloadProtocol {
@ -35,13 +35,19 @@ public abstract class DirectDownloadCommand extends StorageSubSystemCommand {
private PrimaryDataStoreTO destPool;
private String checksum;
private Map<String, String> headers;
private Integer connectTimeout;
private Integer soTimeout;
private Integer connectionRequestTimeout;
protected DirectDownloadCommand (final String url, final Long templateId, final PrimaryDataStoreTO destPool, final String checksum, final Map<String, String> headers) {
protected DirectDownloadCommand (final String url, final Long templateId, final PrimaryDataStoreTO destPool, final String checksum, final Map<String, String> headers, final Integer connectTimeout, final Integer soTimeout, final Integer connectionRequestTimeout) {
this.url = url;
this.templateId = templateId;
this.destPool = destPool;
this.checksum = checksum;
this.headers = headers;
this.connectTimeout = connectTimeout;
this.soTimeout = soTimeout;
this.connectionRequestTimeout = connectionRequestTimeout;
}
public String getUrl() {
@ -64,6 +70,30 @@ public abstract class DirectDownloadCommand extends StorageSubSystemCommand {
return headers;
}
public Integer getConnectTimeout() {
return connectTimeout;
}
public void setConnectTimeout(Integer connectTimeout) {
this.connectTimeout = connectTimeout;
}
public Integer getSoTimeout() {
return soTimeout;
}
public void setSoTimeout(Integer soTimeout) {
this.soTimeout = soTimeout;
}
public Integer getConnectionRequestTimeout() {
return connectionRequestTimeout;
}
public void setConnectionRequestTimeout(Integer connectionRequestTimeout) {
this.connectionRequestTimeout = connectionRequestTimeout;
}
@Override
public void setExecuteInSequence(boolean inSeq) {
}

8
core/src/main/java/org/apache/cloudstack/agent/directdownload/HttpDirectDownloadCommand.java
View File

@ -18,14 +18,14 @@
*/
package org.apache.cloudstack.agent.directdownload;
import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
import java.util.Map;
import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
public class HttpDirectDownloadCommand extends DirectDownloadCommand {
public HttpDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum, Map<String, String> headers) {
super(url, templateId, destPool, checksum, headers);
public HttpDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum, Map<String, String> headers, int connectTimeout, int soTimeout) {
super(url, templateId, destPool, checksum, headers, connectTimeout, soTimeout, null);
}
}

8
core/src/main/java/org/apache/cloudstack/agent/directdownload/HttpsDirectDownloadCommand.java
View File

@ -19,13 +19,13 @@
package org.apache.cloudstack.agent.directdownload;
import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
import java.util.Map;
import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
public class HttpsDirectDownloadCommand extends DirectDownloadCommand {
public HttpsDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum, Map<String, String> headers) {
super(url, templateId, destPool, checksum, headers);
public HttpsDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum, Map<String, String> headers, int connectTimeout, int soTimeout, int connectionRequestTimeout) {
super(url, templateId, destPool, checksum, headers, connectTimeout, soTimeout, connectionRequestTimeout);
}
}

8
core/src/main/java/org/apache/cloudstack/agent/directdownload/MetalinkDirectDownloadCommand.java
View File

@ -18,14 +18,14 @@
//
package org.apache.cloudstack.agent.directdownload;
import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
import java.util.Map;
import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
public class MetalinkDirectDownloadCommand extends DirectDownloadCommand {
public MetalinkDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum, Map<String, String> headers) {
super(url, templateId, destPool, checksum, headers);
public MetalinkDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum, Map<String, String> headers, int connectTimeout, int soTimeout) {
super(url, templateId, destPool, checksum, headers, connectTimeout, soTimeout, null);
}
}

6
core/src/main/java/org/apache/cloudstack/agent/directdownload/NfsDirectDownloadCommand.java
View File

@ -18,14 +18,14 @@
//
package org.apache.cloudstack.agent.directdownload;
import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
import java.util.Map;
import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
public class NfsDirectDownloadCommand extends DirectDownloadCommand {
public NfsDirectDownloadCommand(final String url, final Long templateId, final PrimaryDataStoreTO destPool, final String checksum, final Map<String, String> headers) {
super(url, templateId, destPool, checksum, headers);
super(url, templateId, destPool, checksum, headers, null, null, null);
}
}

4
engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java
View File

@ -345,7 +345,7 @@ public class VMTemplateDaoImpl extends GenericDaoBase<VMTemplateVO, Long> implem
readySystemTemplateSearch.and("state", readySystemTemplateSearch.entity().getState(), SearchCriteria.Op.EQ);
readySystemTemplateSearch.and("templateType", readySystemTemplateSearch.entity().getTemplateType(), SearchCriteria.Op.EQ);
SearchBuilder<TemplateDataStoreVO> templateDownloadSearch = _templateDataStoreDao.createSearchBuilder();
templateDownloadSearch.and("downloadState", templateDownloadSearch.entity().getDownloadState(), SearchCriteria.Op.EQ);
templateDownloadSearch.and("downloadState", templateDownloadSearch.entity().getDownloadState(), SearchCriteria.Op.IN);
readySystemTemplateSearch.join("vmTemplateJoinTemplateStoreRef", templateDownloadSearch, templateDownloadSearch.entity().getTemplateId(),
readySystemTemplateSearch.entity().getId(), JoinBuilder.JoinType.INNER);
SearchBuilder<HostVO> hostHyperSearch2 = _hostDao.createSearchBuilder();
@ -860,7 +860,7 @@ public class VMTemplateDaoImpl extends GenericDaoBase<VMTemplateVO, Long> implem
sc.setParameters("state", VirtualMachineTemplate.State.Active);
sc.setJoinParameters("tmplHyper", "type", Host.Type.Routing);
sc.setJoinParameters("tmplHyper", "zoneId", zoneId);
sc.setJoinParameters("vmTemplateJoinTemplateStoreRef", "downloadState", VMTemplateStorageResourceAssoc.Status.DOWNLOADED);
sc.setJoinParameters("vmTemplateJoinTemplateStoreRef", "downloadState", new VMTemplateStorageResourceAssoc.Status[] {VMTemplateStorageResourceAssoc.Status.DOWNLOADED, VMTemplateStorageResourceAssoc.Status.BYPASSED});
// order by descending order of id
List<VMTemplateVO> tmplts = listBy(sc, new Filter(VMTemplateVO.class, "id", false, null, null));

11
engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateZoneDao.java
View File

@ -22,14 +22,15 @@ import com.cloud.storage.VMTemplateZoneVO;
import com.cloud.utils.db.GenericDao;
public interface VMTemplateZoneDao extends GenericDao<VMTemplateZoneVO, Long> {
public List<VMTemplateZoneVO> listByZoneId(long id);
List<VMTemplateZoneVO> listByZoneId(long id);
public List<VMTemplateZoneVO> listByTemplateId(long templateId);
List<VMTemplateZoneVO> listByTemplateId(long templateId);
public VMTemplateZoneVO findByZoneTemplate(long zoneId, long templateId);
VMTemplateZoneVO findByZoneTemplate(long zoneId, long templateId);
public List<VMTemplateZoneVO> listByZoneTemplate(Long zoneId, long templateId);
List<VMTemplateZoneVO> listByZoneTemplate(Long zoneId, long templateId);
public void deletePrimaryRecordsForTemplate(long templateId);
void deletePrimaryRecordsForTemplate(long templateId);
void deleteByZoneId(long zoneId);
}

9
engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateZoneDaoImpl.java
View File

@ -93,4 +93,13 @@ public class VMTemplateZoneDaoImpl extends GenericDaoBase<VMTemplateZoneVO, Long
txn.commit();
}
@Override
public void deleteByZoneId(long zoneId) {
SearchCriteria<VMTemplateZoneVO> sc = ZoneSearch.create();
sc.setParameters("zone_id", zoneId);
TransactionLegacy txn = TransactionLegacy.currentTxn();
txn.start();
remove(sc);
txn.commit();
}
}

7
engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java
View File

@ -264,13 +264,13 @@ public class TemplateServiceImpl implements TemplateService {
List<VMTemplateVO> defaultBuiltin = _templateDao.listDefaultBuiltinTemplates();
for (VMTemplateVO rtngTmplt : rtngTmplts) {
if (rtngTmplt.getHypervisorType() == hostHyper) {
if (rtngTmplt.getHypervisorType() == hostHyper && !rtngTmplt.isDirectDownload()) {
toBeDownloaded.add(rtngTmplt);
}
}
for (VMTemplateVO builtinTmplt : defaultBuiltin) {
if (builtinTmplt.getHypervisorType() == hostHyper) {
if (builtinTmplt.getHypervisorType() == hostHyper && !builtinTmplt.isDirectDownload()) {
toBeDownloaded.add(builtinTmplt);
}
}
@ -1201,6 +1201,9 @@ public class TemplateServiceImpl implements TemplateService {
for (VMTemplateVO tmplt : rtngTmplts) {
TemplateDataStoreVO tmpltStore = _vmTemplateStoreDao.findByStoreTemplate(storeId, tmplt.getId());
if (tmpltStore == null) {
if (_vmTemplateStoreDao.isTemplateMarkedForDirectDownload(tmplt.getId())) {
continue;
}
tmpltStore =
new TemplateDataStoreVO(storeId, tmplt.getId(), new Date(), 100, Status.DOWNLOADED, null, null, null,
TemplateConstants.DEFAULT_SYSTEM_VM_TEMPLATE_PATH + tmplt.getId() + '/', tmplt.getUrl());

6
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java
View File

@ -1678,13 +1678,13 @@ public class KVMStorageProcessor implements StorageProcessor {
*/
private DirectTemplateDownloader getDirectTemplateDownloaderFromCommand(DirectDownloadCommand cmd, KVMStoragePool destPool) {
if (cmd instanceof HttpDirectDownloadCommand) {
return new HttpDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPool.getLocalPath(), cmd.getChecksum(), cmd.getHeaders());
return new HttpDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPool.getLocalPath(), cmd.getChecksum(), cmd.getHeaders(), cmd.getConnectTimeout(), cmd.getSoTimeout());
} else if (cmd instanceof HttpsDirectDownloadCommand) {
return new HttpsDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPool.getLocalPath(), cmd.getChecksum(), cmd.getHeaders());
return new HttpsDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPool.getLocalPath(), cmd.getChecksum(), cmd.getHeaders(), cmd.getConnectTimeout(), cmd.getSoTimeout(), cmd.getConnectionRequestTimeout());
} else if (cmd instanceof NfsDirectDownloadCommand) {
return new NfsDirectTemplateDownloader(cmd.getUrl(), destPool.getLocalPath(), cmd.getTemplateId(), cmd.getChecksum());
} else if (cmd instanceof MetalinkDirectDownloadCommand) {
return new MetalinkDirectTemplateDownloader(cmd.getUrl(), destPool.getLocalPath(), cmd.getTemplateId(), cmd.getChecksum(), cmd.getHeaders());
return new MetalinkDirectTemplateDownloader(cmd.getUrl(), destPool.getLocalPath(), cmd.getTemplateId(), cmd.getChecksum(), cmd.getHeaders(), cmd.getConnectTimeout(), cmd.getSoTimeout());
} else {
throw new IllegalArgumentException("Unsupported protocol, please provide HTTP(S), NFS or a metalink");
}

185
server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java
View File

@ -16,6 +16,96 @@
// under the License.
package com.cloud.configuration;
import java.io.UnsupportedEncodingException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URLDecoder;
import java.sql.Date;
import java.sql.PreparedStatement;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.ListIterator;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.UUID;
import javax.inject.Inject;
import javax.naming.ConfigurationException;
import org.apache.cloudstack.acl.SecurityChecker;
import org.apache.cloudstack.affinity.AffinityGroup;
import org.apache.cloudstack.affinity.AffinityGroupService;
import org.apache.cloudstack.affinity.dao.AffinityGroupDao;
import org.apache.cloudstack.agent.lb.IndirectAgentLB;
import org.apache.cloudstack.agent.lb.IndirectAgentLBServiceImpl;
import org.apache.cloudstack.api.ApiConstants;
import org.apache.cloudstack.api.command.admin.config.UpdateCfgCmd;
import org.apache.cloudstack.api.command.admin.network.CreateManagementNetworkIpRangeCmd;
import org.apache.cloudstack.api.command.admin.network.CreateNetworkOfferingCmd;
import org.apache.cloudstack.api.command.admin.network.DeleteManagementNetworkIpRangeCmd;
import org.apache.cloudstack.api.command.admin.network.DeleteNetworkOfferingCmd;
import org.apache.cloudstack.api.command.admin.network.UpdateNetworkOfferingCmd;
import org.apache.cloudstack.api.command.admin.offering.CreateDiskOfferingCmd;
import org.apache.cloudstack.api.command.admin.offering.CreateServiceOfferingCmd;
import org.apache.cloudstack.api.command.admin.offering.DeleteDiskOfferingCmd;
import org.apache.cloudstack.api.command.admin.offering.DeleteServiceOfferingCmd;
import org.apache.cloudstack.api.command.admin.offering.UpdateDiskOfferingCmd;
import org.apache.cloudstack.api.command.admin.offering.UpdateServiceOfferingCmd;
import org.apache.cloudstack.api.command.admin.pod.DeletePodCmd;
import org.apache.cloudstack.api.command.admin.pod.UpdatePodCmd;
import org.apache.cloudstack.api.command.admin.region.CreatePortableIpRangeCmd;
import org.apache.cloudstack.api.command.admin.region.DeletePortableIpRangeCmd;
import org.apache.cloudstack.api.command.admin.region.ListPortableIpRangesCmd;
import org.apache.cloudstack.api.command.admin.vlan.CreateVlanIpRangeCmd;
import org.apache.cloudstack.api.command.admin.vlan.DedicatePublicIpRangeCmd;
import org.apache.cloudstack.api.command.admin.vlan.DeleteVlanIpRangeCmd;
import org.apache.cloudstack.api.command.admin.vlan.ReleasePublicIpRangeCmd;
import org.apache.cloudstack.api.command.admin.zone.CreateZoneCmd;
import org.apache.cloudstack.api.command.admin.zone.DeleteZoneCmd;
import org.apache.cloudstack.api.command.admin.zone.UpdateZoneCmd;
import org.apache.cloudstack.api.command.user.network.ListNetworkOfferingsCmd;
import org.apache.cloudstack.config.ApiServiceConfiguration;
import org.apache.cloudstack.config.Configuration;
import org.apache.cloudstack.context.CallContext;
import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope;
import org.apache.cloudstack.framework.config.ConfigDepot;
import org.apache.cloudstack.framework.config.ConfigKey;
import org.apache.cloudstack.framework.config.Configurable;
import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
import org.apache.cloudstack.framework.config.impl.ConfigurationVO;
import org.apache.cloudstack.framework.messagebus.MessageBus;
import org.apache.cloudstack.framework.messagebus.MessageSubscriber;
import org.apache.cloudstack.framework.messagebus.PublishScope;
import org.apache.cloudstack.region.PortableIp;
import org.apache.cloudstack.region.PortableIpDao;
import org.apache.cloudstack.region.PortableIpRange;
import org.apache.cloudstack.region.PortableIpRangeDao;
import org.apache.cloudstack.region.PortableIpRangeVO;
import org.apache.cloudstack.region.PortableIpVO;
import org.apache.cloudstack.region.Region;
import org.apache.cloudstack.region.RegionVO;
import org.apache.cloudstack.region.dao.RegionDao;
import org.apache.cloudstack.resourcedetail.DiskOfferingDetailVO;
import org.apache.cloudstack.resourcedetail.dao.DiskOfferingDetailsDao;
import org.apache.cloudstack.storage.datastore.db.ImageStoreDao;
import org.apache.cloudstack.storage.datastore.db.ImageStoreDetailsDao;
import org.apache.cloudstack.storage.datastore.db.ImageStoreVO;
import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao;
import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.log4j.Logger;
import com.cloud.agent.AgentManager;
import com.cloud.alert.AlertManager;
import com.cloud.api.ApiDBUtils;
@ -118,6 +208,7 @@ import com.cloud.storage.DiskOfferingVO;
import com.cloud.storage.Storage.ProvisioningType;
import com.cloud.storage.StorageManager;
import com.cloud.storage.dao.DiskOfferingDao;
import com.cloud.storage.dao.VMTemplateZoneDao;
import com.cloud.storage.dao.VolumeDao;
import com.cloud.test.IPRangeConfig;
import com.cloud.user.Account;
@ -158,95 +249,6 @@ import com.google.common.base.MoreObjects;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.collect.Sets;
import org.apache.cloudstack.acl.SecurityChecker;
import org.apache.cloudstack.affinity.AffinityGroup;
import org.apache.cloudstack.affinity.AffinityGroupService;
import org.apache.cloudstack.affinity.dao.AffinityGroupDao;
import org.apache.cloudstack.agent.lb.IndirectAgentLB;
import org.apache.cloudstack.agent.lb.IndirectAgentLBServiceImpl;
import org.apache.cloudstack.api.ApiConstants;
import org.apache.cloudstack.api.command.admin.config.UpdateCfgCmd;
import org.apache.cloudstack.api.command.admin.network.CreateManagementNetworkIpRangeCmd;
import org.apache.cloudstack.api.command.admin.network.CreateNetworkOfferingCmd;
import org.apache.cloudstack.api.command.admin.network.DeleteManagementNetworkIpRangeCmd;
import org.apache.cloudstack.api.command.admin.network.DeleteNetworkOfferingCmd;
import org.apache.cloudstack.api.command.admin.network.UpdateNetworkOfferingCmd;
import org.apache.cloudstack.api.command.admin.offering.CreateDiskOfferingCmd;
import org.apache.cloudstack.api.command.admin.offering.CreateServiceOfferingCmd;
import org.apache.cloudstack.api.command.admin.offering.DeleteDiskOfferingCmd;
import org.apache.cloudstack.api.command.admin.offering.DeleteServiceOfferingCmd;
import org.apache.cloudstack.api.command.admin.offering.UpdateDiskOfferingCmd;
import org.apache.cloudstack.api.command.admin.offering.UpdateServiceOfferingCmd;
import org.apache.cloudstack.api.command.admin.pod.DeletePodCmd;
import org.apache.cloudstack.api.command.admin.pod.UpdatePodCmd;
import org.apache.cloudstack.api.command.admin.region.CreatePortableIpRangeCmd;
import org.apache.cloudstack.api.command.admin.region.DeletePortableIpRangeCmd;
import org.apache.cloudstack.api.command.admin.region.ListPortableIpRangesCmd;
import org.apache.cloudstack.api.command.admin.vlan.CreateVlanIpRangeCmd;
import org.apache.cloudstack.api.command.admin.vlan.DedicatePublicIpRangeCmd;
import org.apache.cloudstack.api.command.admin.vlan.DeleteVlanIpRangeCmd;
import org.apache.cloudstack.api.command.admin.vlan.ReleasePublicIpRangeCmd;
import org.apache.cloudstack.api.command.admin.zone.CreateZoneCmd;
import org.apache.cloudstack.api.command.admin.zone.DeleteZoneCmd;
import org.apache.cloudstack.api.command.admin.zone.UpdateZoneCmd;
import org.apache.cloudstack.api.command.user.network.ListNetworkOfferingsCmd;
import org.apache.cloudstack.config.ApiServiceConfiguration;
import org.apache.cloudstack.config.Configuration;
import org.apache.cloudstack.context.CallContext;
import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope;
import org.apache.cloudstack.framework.config.ConfigDepot;
import org.apache.cloudstack.framework.config.ConfigKey;
import org.apache.cloudstack.framework.config.Configurable;
import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
import org.apache.cloudstack.framework.config.impl.ConfigurationVO;
import org.apache.cloudstack.framework.messagebus.MessageBus;
import org.apache.cloudstack.framework.messagebus.MessageSubscriber;
import org.apache.cloudstack.framework.messagebus.PublishScope;
import org.apache.cloudstack.region.PortableIp;
import org.apache.cloudstack.region.PortableIpDao;
import org.apache.cloudstack.region.PortableIpRange;
import org.apache.cloudstack.region.PortableIpRangeDao;
import org.apache.cloudstack.region.PortableIpRangeVO;
import org.apache.cloudstack.region.PortableIpVO;
import org.apache.cloudstack.region.Region;
import org.apache.cloudstack.region.RegionVO;
import org.apache.cloudstack.region.dao.RegionDao;
import org.apache.cloudstack.resourcedetail.DiskOfferingDetailVO;
import org.apache.cloudstack.resourcedetail.dao.DiskOfferingDetailsDao;
import org.apache.cloudstack.storage.datastore.db.ImageStoreDao;
import org.apache.cloudstack.storage.datastore.db.ImageStoreDetailsDao;
import org.apache.cloudstack.storage.datastore.db.ImageStoreVO;
import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao;
import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.log4j.Logger;
import java.io.UnsupportedEncodingException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URLDecoder;
import java.sql.Date;
import java.sql.PreparedStatement;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.ListIterator;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.UUID;
import javax.inject.Inject;
import javax.naming.ConfigurationException;
public class ConfigurationManagerImpl extends ManagerBase implements ConfigurationManager, ConfigurationService, Configurable {
public static final Logger s_logger = Logger.getLogger(ConfigurationManagerImpl.class);
@ -381,7 +383,8 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
AgentManager _agentManager;
@Inject
IndirectAgentLB _indirectAgentLB;
@Inject
private VMTemplateZoneDao templateZoneDao;
// FIXME - why don't we have interface for DataCenterLinkLocalIpAddressDao?
@Inject
@ -1838,6 +1841,8 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
final boolean success = _zoneDao.remove(zoneId);
if (success) {
// delete template refs for this zone
templateZoneDao.deleteByZoneId(zoneId);
// delete all capacity records for the zone
_capacityDao.removeBy(null, zoneId, null, null, null);
// remove from dedicated resources

8
server/src/main/java/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java
View File

@ -100,6 +100,7 @@ import com.cloud.resource.ServerResource;
import com.cloud.resource.UnableDeleteHostException;
import com.cloud.service.ServiceOfferingVO;
import com.cloud.service.dao.ServiceOfferingDao;
import com.cloud.storage.DataStoreRole;
import com.cloud.storage.Storage;
import com.cloud.storage.StoragePoolStatus;
import com.cloud.storage.VMTemplateStorageResourceAssoc.Status;
@ -1011,7 +1012,12 @@ public class ConsoleProxyManagerImpl extends ManagerBase implements ConsoleProxy
}
return false;
}
TemplateDataStoreVO templateHostRef = _vmTemplateStoreDao.findByTemplateZoneDownloadStatus(template.getId(), dataCenterId, Status.DOWNLOADED);
TemplateDataStoreVO templateHostRef = null;
if (template.isDirectDownload()) {
templateHostRef = _vmTemplateStoreDao.findByTemplate(template.getId(), DataStoreRole.Image);
} else {
templateHostRef = _vmTemplateStoreDao.findByTemplateZoneDownloadStatus(template.getId(), dataCenterId, Status.DOWNLOADED);
}
if (templateHostRef != null) {
boolean useLocalStorage = false;

109
server/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadManagerImpl.java
View File

@ -20,6 +20,53 @@ package org.apache.cloudstack.direct.download;
import static com.cloud.storage.Storage.ImageFormat;
import java.net.URI;
import java.net.URISyntaxException;
import java.security.cert.Certificate;
import java.security.cert.CertificateException;
import java.security.cert.CertificateExpiredException;
import java.security.cert.CertificateNotYetValidException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import javax.inject.Inject;
import javax.naming.ConfigurationException;
import org.apache.cloudstack.agent.directdownload.DirectDownloadAnswer;
import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand;
import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand.DownloadProtocol;
import org.apache.cloudstack.agent.directdownload.HttpDirectDownloadCommand;
import org.apache.cloudstack.agent.directdownload.HttpsDirectDownloadCommand;
import org.apache.cloudstack.agent.directdownload.MetalinkDirectDownloadCommand;
import org.apache.cloudstack.agent.directdownload.NfsDirectDownloadCommand;
import org.apache.cloudstack.agent.directdownload.RevokeDirectDownloadCertificateCommand;
import org.apache.cloudstack.agent.directdownload.SetupDirectDownloadCertificateCommand;
import org.apache.cloudstack.context.CallContext;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine;
import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStore;
import org.apache.cloudstack.framework.config.ConfigKey;
import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
import org.apache.cloudstack.managed.context.ManagedContextRunnable;
import org.apache.cloudstack.poll.BackgroundPollManager;
import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.log4j.Logger;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import com.cloud.agent.AgentManager;
import com.cloud.agent.api.Answer;
import com.cloud.dc.DataCenterVO;
@ -43,52 +90,8 @@ import com.cloud.storage.dao.VMTemplatePoolDao;
import com.cloud.utils.component.ManagerBase;
import com.cloud.utils.concurrency.NamedThreadFactory;
import com.cloud.utils.exception.CloudRuntimeException;
import java.net.URI;
import java.net.URISyntaxException;
import java.security.cert.Certificate;
import java.security.cert.CertificateException;
import java.security.cert.CertificateExpiredException;
import java.security.cert.CertificateNotYetValidException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Arrays;
import java.util.Collections;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import javax.inject.Inject;
import javax.naming.ConfigurationException;
import com.cloud.utils.security.CertificateHelper;
import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand;
import org.apache.cloudstack.agent.directdownload.DirectDownloadAnswer;
import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand.DownloadProtocol;
import org.apache.cloudstack.agent.directdownload.HttpDirectDownloadCommand;
import org.apache.cloudstack.agent.directdownload.MetalinkDirectDownloadCommand;
import org.apache.cloudstack.agent.directdownload.NfsDirectDownloadCommand;
import org.apache.cloudstack.agent.directdownload.HttpsDirectDownloadCommand;
import org.apache.cloudstack.agent.directdownload.RevokeDirectDownloadCertificateCommand;
import org.apache.cloudstack.agent.directdownload.SetupDirectDownloadCertificateCommand;
import org.apache.cloudstack.context.CallContext;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine;
import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStore;
import org.apache.cloudstack.framework.config.ConfigKey;
import org.apache.cloudstack.managed.context.ManagedContextRunnable;
import org.apache.cloudstack.poll.BackgroundPollManager;
import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.log4j.Logger;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import sun.security.x509.X509CertImpl;
public class DirectDownloadManagerImpl extends ManagerBase implements DirectDownloadManager {
@ -119,6 +122,8 @@ public class DirectDownloadManagerImpl extends ManagerBase implements DirectDown
private BackgroundPollManager backgroundPollManager;
@Inject
private DataCenterDao dataCenterDao;
@Inject
private ConfigurationDao configDao;
protected ScheduledExecutorService executorService;
@ -320,14 +325,17 @@ public class DirectDownloadManagerImpl extends ManagerBase implements DirectDown
*/
private DirectDownloadCommand getDirectDownloadCommandFromProtocol(DownloadProtocol protocol, String url, Long templateId, PrimaryDataStoreTO destPool,
String checksum, Map<String, String> httpHeaders) {
int connectTimeout = DirectDownloadConnectTimeout.value();
int soTimeout = DirectDownloadSocketTimeout.value();
int connectionRequestTimeout = DirectDownloadConnectionRequestTimeout.value();
if (protocol.equals(DownloadProtocol.HTTP)) {
return new HttpDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders);
return new HttpDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders, connectTimeout, soTimeout);
} else if (protocol.equals(DownloadProtocol.HTTPS)) {
return new HttpsDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders);
return new HttpsDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders, connectTimeout, soTimeout, connectionRequestTimeout);
} else if (protocol.equals(DownloadProtocol.NFS)) {
return new NfsDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders);
} else if (protocol.equals(DownloadProtocol.METALINK)) {
return new MetalinkDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders);
return new MetalinkDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders, connectTimeout, soTimeout);
} else {
return null;
}
@ -549,7 +557,10 @@ public class DirectDownloadManagerImpl extends ManagerBase implements DirectDown
@Override
public ConfigKey<?>[] getConfigKeys() {
return new ConfigKey<?>[]{
DirectDownloadCertificateUploadInterval
DirectDownloadCertificateUploadInterval,
DirectDownloadConnectTimeout,
DirectDownloadSocketTimeout,
DirectDownloadConnectionRequestTimeout
};
}

1
server/src/test/resources/createNetworkOffering.xml
View File

@ -58,5 +58,6 @@
<bean id="DiskOfferingDetailsDaoImpl" class="org.apache.cloudstack.resourcedetail.dao.DiskOfferingDetailsDaoImpl" />
<bean id="networkOfferingJoinDaoImpl" class="com.cloud.api.query.dao.NetworkOfferingJoinDaoImpl" />
<bean id="networkOfferingDetailsDaoImpl" class="com.cloud.offerings.dao.NetworkOfferingDetailsDaoImpl" />
<bean id="vMTemplateZoneDaoImpl" class="com.cloud.storage.dao.VMTemplateZoneDaoImpl" />
<bean id="indirectAgentLBImpl" class="org.apache.cloudstack.agent.lb.IndirectAgentLBServiceImpl" />
</beans>

3
services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java
View File

@ -828,8 +828,7 @@ public class SecondaryStorageManagerImpl extends ManagerBase implements Secondar
return false;
}
DataStore store = templateMgr.getImageStore(dataCenterId, template.getId());
if (store == null) {
if (!template.isDirectDownload() && templateMgr.getImageStore(dataCenterId, template.getId()) == null) {
if (s_logger.isDebugEnabled()) {
s_logger.debug("No secondary storage available in zone " + dataCenterId + ", wait until it is ready to launch secondary storage vm");
}