From 5e2f02fdd57258aa5efec5cdf680bdb3fbfce738 Mon Sep 17 00:00:00 2001 From: Edison Su Date: Thu, 26 Jan 2012 18:45:31 -0800 Subject: [PATCH] bug 13315: add httpproxy support in ssvm, only basic auth. User can add secstorage.proxy = http://username:password@hostname:port status 13315: resolved fixed --- .../agent/api/storage/DownloadCommand.java | 61 +++++++++++++++++++ .../storage/template/DownloadManager.java | 3 +- .../storage/template/DownloadManagerImpl.java | 7 ++- .../template/HttpTemplateDownloader.java | 14 ++++- .../src/com/cloud/configuration/Config.java | 2 + .../storage/download/DownloadMonitorImpl.java | 22 ++++++- .../SecondaryStorageManagerImpl.java | 28 +++++++++ 7 files changed, 129 insertions(+), 8 deletions(-) diff --git a/api/src/com/cloud/agent/api/storage/DownloadCommand.java b/api/src/com/cloud/agent/api/storage/DownloadCommand.java index a0dfb9ee9cd..375c13400f5 100644 --- a/api/src/com/cloud/agent/api/storage/DownloadCommand.java +++ b/api/src/com/cloud/agent/api/storage/DownloadCommand.java @@ -17,6 +17,8 @@ */ package com.cloud.agent.api.storage; +import java.net.URI; + import com.cloud.storage.Storage.ImageFormat; import com.cloud.template.VirtualMachineTemplate; @@ -43,10 +45,61 @@ public class DownloadCommand extends AbstractDownloadCommand { return password; } } + + public static class Proxy { + private String _host; + private int _port; + private String _userName; + private String _password; + + public Proxy() { + + } + + public Proxy(String host, int port, String userName, String password) { + this._host = host; + this._port = port; + this._userName = userName; + this._password = password; + } + + public Proxy(URI uri) { + this._host = uri.getHost(); + this._port = uri.getPort() == -1 ? 3128 : uri.getPort(); + String userInfo = uri.getUserInfo(); + if (userInfo != null) { + String[] tokens = userInfo.split(":"); + if (tokens.length == 1) { + this._userName = userInfo; + this._password = ""; + } else if (tokens.length == 2) { + this._userName = tokens[0]; + this._password = tokens[1]; + } + } + } + + public String getHost() { + return _host; + } + + public int getPort() { + return _port; + } + + public String getUserName() { + return _userName; + } + + public String getPassword() { + return _password; + } + } private boolean hvm; private String description; private String checksum; private PasswordAuth auth; + private Proxy _proxy; private Long maxDownloadSizeInBytes = null; private long id; @@ -128,6 +181,14 @@ public class DownloadCommand extends AbstractDownloadCommand { auth = new PasswordAuth(userName, passwd); } + public Proxy getProxy() { + return _proxy; + } + + public void setProxy(Proxy proxy) { + _proxy = proxy; + } + public Long getMaxDownloadSizeInBytes() { return maxDownloadSizeInBytes; } diff --git a/core/src/com/cloud/storage/template/DownloadManager.java b/core/src/com/cloud/storage/template/DownloadManager.java index e13894dd9e5..63e45dc6ef1 100644 --- a/core/src/com/cloud/storage/template/DownloadManager.java +++ b/core/src/com/cloud/storage/template/DownloadManager.java @@ -22,6 +22,7 @@ import java.util.Map; import com.cloud.agent.api.storage.DownloadAnswer; import com.cloud.agent.api.storage.DownloadCommand; +import com.cloud.agent.api.storage.DownloadCommand.Proxy; import com.cloud.storage.VMTemplateHostVO; import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.resource.SecondaryStorageResource; @@ -49,7 +50,7 @@ public interface DownloadManager extends Manager { * @param maxDownloadSizeInBytes (optional) max download size for the template, in bytes. * @return job-id that can be used to interrogate the status of the download. */ - public String downloadPublicTemplate(long id, String url, String name, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum, String installPathPrefix, String userName, String passwd, long maxDownloadSizeInBytes); + public String downloadPublicTemplate(long id, String url, String name, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum, String installPathPrefix, String userName, String passwd, long maxDownloadSizeInBytes, Proxy proxy); /** diff --git a/core/src/com/cloud/storage/template/DownloadManagerImpl.java b/core/src/com/cloud/storage/template/DownloadManagerImpl.java index 21e0a67f79e..fea65c6287c 100755 --- a/core/src/com/cloud/storage/template/DownloadManagerImpl.java +++ b/core/src/com/cloud/storage/template/DownloadManagerImpl.java @@ -47,6 +47,7 @@ import org.apache.log4j.Logger; import com.cloud.agent.api.Answer; import com.cloud.agent.api.storage.DownloadAnswer; import com.cloud.agent.api.storage.DownloadCommand; +import com.cloud.agent.api.storage.DownloadCommand.Proxy; import com.cloud.agent.api.storage.DownloadProgressCommand; import com.cloud.agent.api.storage.DownloadProgressCommand.RequestType; import com.cloud.exception.InternalErrorException; @@ -421,7 +422,7 @@ public class DownloadManagerImpl implements DownloadManager { } @Override - public String downloadPublicTemplate(long id, String url, String name, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum, String installPathPrefix, String user, String password, long maxTemplateSizeInBytes) { + public String downloadPublicTemplate(long id, String url, String name, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum, String installPathPrefix, String user, String password, long maxTemplateSizeInBytes, Proxy proxy) { UUID uuid = UUID.randomUUID(); String jobId = uuid.toString(); String tmpDir = installPathPrefix + File.separator + accountId + File.separator + id; @@ -452,7 +453,7 @@ public class DownloadManagerImpl implements DownloadManager { TemplateDownloader td; if ((uri != null) && (uri.getScheme() != null)) { if (uri.getScheme().equalsIgnoreCase("http") || uri.getScheme().equalsIgnoreCase("https")) { - td = new HttpTemplateDownloader(_storage, url, tmpDir, new Completion(jobId), maxTemplateSizeInBytes, user, password); + td = new HttpTemplateDownloader(_storage, url, tmpDir, new Completion(jobId), maxTemplateSizeInBytes, user, password, proxy); } else if (uri.getScheme().equalsIgnoreCase("file")) { td = new LocalTemplateDownloader(_storage, url, tmpDir, maxTemplateSizeInBytes, new Completion(jobId)); } else if (uri.getScheme().equalsIgnoreCase("scp")) { @@ -582,7 +583,7 @@ public class DownloadManagerImpl implements DownloadManager { } long maxDownloadSizeInBytes = (cmd.getMaxDownloadSizeInBytes() == null) ? TemplateDownloader.DEFAULT_MAX_TEMPLATE_SIZE_IN_BYTES : (cmd.getMaxDownloadSizeInBytes()); - String jobId = downloadPublicTemplate(cmd.getId(), cmd.getUrl(), cmd.getName(), cmd.getFormat(), cmd.isHvm(), cmd.getAccountId(), cmd.getDescription(), cmd.getChecksum(), installPathPrefix, user, password, maxDownloadSizeInBytes); + String jobId = downloadPublicTemplate(cmd.getId(), cmd.getUrl(), cmd.getName(), cmd.getFormat(), cmd.isHvm(), cmd.getAccountId(), cmd.getDescription(), cmd.getChecksum(), installPathPrefix, user, password, maxDownloadSizeInBytes, cmd.getProxy()); sleep(); if (jobId == null) { return new DownloadAnswer("Internal Error", VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR); diff --git a/core/src/com/cloud/storage/template/HttpTemplateDownloader.java b/core/src/com/cloud/storage/template/HttpTemplateDownloader.java index 8ffb0c1308b..3da1aaae21b 100644 --- a/core/src/com/cloud/storage/template/HttpTemplateDownloader.java +++ b/core/src/com/cloud/storage/template/HttpTemplateDownloader.java @@ -45,6 +45,7 @@ import org.apache.commons.httpclient.methods.GetMethod; import org.apache.commons.httpclient.params.HttpMethodParams; import org.apache.log4j.Logger; +import com.cloud.agent.api.storage.DownloadCommand.Proxy; import com.cloud.storage.StorageLayer; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.Pair; @@ -77,7 +78,7 @@ public class HttpTemplateDownloader implements TemplateDownloader { private final HttpMethodRetryHandler myretryhandler; - public HttpTemplateDownloader (StorageLayer storageLayer, String downloadUrl, String toDir, DownloadCompleteCallback callback, long maxTemplateSizeInBytes, String user, String password) { + public HttpTemplateDownloader (StorageLayer storageLayer, String downloadUrl, String toDir, DownloadCompleteCallback callback, long maxTemplateSizeInBytes, String user, String password, Proxy proxy) { this._storage = storageLayer; this.downloadUrl = downloadUrl; this.setToDir(toDir); @@ -125,7 +126,14 @@ public class HttpTemplateDownloader implements TemplateDownloader { toFile = f.getAbsolutePath(); Pair hostAndPort = validateUrl(downloadUrl); - + + if (proxy != null) { + client.getHostConfiguration().setProxy(proxy.getHost(), proxy.getPort()); + if (proxy.getUserName() != null) { + Credentials proxyCreds = new UsernamePasswordCredentials(proxy.getUserName(), proxy.getPassword()); + client.getState().setProxyCredentials(AuthScope.ANY, proxyCreds); + } + } if ((user != null) && (password != null)) { client.getParams().setAuthenticationPreemptive(true); Credentials defaultcreds = new UsernamePasswordCredentials(user, password); @@ -425,7 +433,7 @@ public class HttpTemplateDownloader implements TemplateDownloader { // TODO Auto-generated catch block e.printStackTrace(); } - TemplateDownloader td = new HttpTemplateDownloader(null, url,"/tmp/mysql", null, TemplateDownloader.DEFAULT_MAX_TEMPLATE_SIZE_IN_BYTES, null, null); + TemplateDownloader td = new HttpTemplateDownloader(null, url,"/tmp/mysql", null, TemplateDownloader.DEFAULT_MAX_TEMPLATE_SIZE_IN_BYTES, null, null, null); long bytes = td.download(true, null); if (bytes > 0) { System.out.println("Downloaded (" + bytes + " bytes)" + " in " + td.getDownloadTime()/1000 + " secs"); diff --git a/server/src/com/cloud/configuration/Config.java b/server/src/com/cloud/configuration/Config.java index ae890e6b574..dc280469ef8 100755 --- a/server/src/com/cloud/configuration/Config.java +++ b/server/src/com/cloud/configuration/Config.java @@ -201,6 +201,8 @@ public enum Config { SecStorageCapacityStandby("Advanced", AgentManager.class, Integer.class, "secstorage.capacity.standby", "10", "The minimal number of command execution sessions that system is able to serve immediately(standby capacity)", null), SecStorageSessionMax("Advanced", AgentManager.class, Integer.class, "secstorage.session.max", "50", "The max number of command execution sessions that a SSVM can handle", null), SecStorageCmdExecutionTimeMax("Advanced", AgentManager.class, Integer.class, "secstorage.cmd.execution.time.max", "30", "The max command execution time in minute", null), + SecStorageProxy("Advanced", AgentManager.class, String.class, "secstorage.proxy", null, "http proxy used by ssvm, in http://username:password@proxyserver:port format", null), + DirectAttachNetworkEnabled("Advanced", ManagementServer.class, Boolean.class, "direct.attach.network.externalIpAllocator.enabled", "false", "Direct-attach VMs using external DHCP server", "true,false"), DirectAttachNetworkExternalAPIURL("Advanced", ManagementServer.class, String.class, "direct.attach.network.externalIpAllocator.url", null, "Direct-attach VMs using external DHCP server (API url)", null), diff --git a/server/src/com/cloud/storage/download/DownloadMonitorImpl.java b/server/src/com/cloud/storage/download/DownloadMonitorImpl.java index 891cfa280af..8c015784537 100755 --- a/server/src/com/cloud/storage/download/DownloadMonitorImpl.java +++ b/server/src/com/cloud/storage/download/DownloadMonitorImpl.java @@ -17,6 +17,8 @@ */ package com.cloud.storage.download; +import java.net.URI; +import java.net.URISyntaxException; import java.util.ArrayList; import java.util.Date; import java.util.HashSet; @@ -36,12 +38,14 @@ import com.cloud.agent.api.Answer; import com.cloud.agent.api.Command; import com.cloud.agent.api.storage.DeleteTemplateCommand; import com.cloud.agent.api.storage.DownloadCommand; +import com.cloud.agent.api.storage.DownloadCommand.Proxy; import com.cloud.agent.api.storage.DownloadProgressCommand; import com.cloud.agent.api.storage.DownloadProgressCommand.RequestType; import com.cloud.agent.api.storage.ListTemplateAnswer; import com.cloud.agent.api.storage.ListTemplateCommand; import com.cloud.agent.manager.Commands; import com.cloud.alert.AlertManager; +import com.cloud.configuration.Config; import com.cloud.configuration.dao.ConfigurationDao; import com.cloud.dc.DataCenterVO; import com.cloud.dc.dao.ClusterDao; @@ -146,6 +150,7 @@ public class DownloadMonitorImpl implements DownloadMonitor { private String _name; private Boolean _sslCopy = new Boolean(false); private String _copyAuthPasswd; + private String _proxy = null; protected SearchBuilder ReadyTemplateStatesSearch; Timer _timer; @@ -162,6 +167,7 @@ public class DownloadMonitorImpl implements DownloadMonitor { _name = name; final Map configs = _configDao.getConfiguration("ManagementServer", params); _sslCopy = Boolean.parseBoolean(configs.get("secstorage.encrypt.copy")); + _proxy = configs.get(Config.SecStorageProxy.key()); String cert = configs.get("secstorage.ssl.cert.domain"); if (!"realhostip.com".equalsIgnoreCase(cert)) { @@ -250,7 +256,7 @@ public class DownloadMonitorImpl implements DownloadMonitor { String sourceChecksum = _vmMgr.getChecksum(srcTmpltHost.getHostId(), srcTmpltHost.getInstallPath()); DownloadCommand dcmd = new DownloadCommand(destServer.getStorageUrl(), url, template, TemplateConstants.DEFAULT_HTTP_AUTH_USER, _copyAuthPasswd, maxTemplateSizeInBytes); - + dcmd.setProxy(getHttpProxy()); if (downloadJobExists) { dcmd = new DownloadProgressCommand(dcmd, destTmpltHost.getJobId(), RequestType.GET_OR_RESTART); } @@ -335,6 +341,7 @@ public class DownloadMonitorImpl implements DownloadMonitor { start(); DownloadCommand dcmd = new DownloadCommand(secUrl, template, maxTemplateSizeInBytes); + dcmd.setProxy(getHttpProxy()); if (downloadJobExists) { dcmd = new DownloadProgressCommand(dcmd, vmTemplateHost.getJobId(), RequestType.GET_OR_RESTART); } @@ -750,5 +757,18 @@ public class DownloadMonitorImpl implements DownloadMonitor { } } + private Proxy getHttpProxy() { + if (_proxy == null) { + return null; + } + try { + URI uri = new URI(_proxy); + Proxy prx = new Proxy(uri); + return prx; + } catch (URISyntaxException e) { + return null; + } + } + } diff --git a/server/src/com/cloud/storage/secondary/SecondaryStorageManagerImpl.java b/server/src/com/cloud/storage/secondary/SecondaryStorageManagerImpl.java index 0cc8e217cf5..0d3802a5e2a 100755 --- a/server/src/com/cloud/storage/secondary/SecondaryStorageManagerImpl.java +++ b/server/src/com/cloud/storage/secondary/SecondaryStorageManagerImpl.java @@ -17,6 +17,8 @@ */ package com.cloud.storage.secondary; +import java.net.URI; +import java.net.URISyntaxException; import java.util.ArrayList; import java.util.Collections; import java.util.Date; @@ -225,6 +227,7 @@ public class SecondaryStorageManagerImpl implements SecondaryStorageVmManager, V private String _instance; private boolean _useLocalStorage; private boolean _useSSlCopy; + private String _httpProxy; private String _allowedInternalSites; protected long _nodeId = ManagementServerNode.getManagementServerId(); @@ -833,6 +836,31 @@ public class SecondaryStorageManagerImpl implements SecondaryStorageVmManager, V _loadScanner = new SystemVmLoadScanner(this); _loadScanner.initScan(STARTUP_DELAY, _capacityScanInterval); } + + _httpProxy = configs.get(Config.SecStorageProxy.key()); + if (_httpProxy != null) { + boolean valid = true; + String errMsg = null; + try { + URI uri = new URI(_httpProxy); + if (!"http".equalsIgnoreCase(uri.getScheme())) { + errMsg = "Only support http proxy"; + valid = false; + } else if (uri.getHost() == null) { + errMsg = "host can not be null"; + valid = false; + } else if (uri.getPort() == -1) { + _httpProxy = _httpProxy + ":3128"; + } + } catch (URISyntaxException e) { + errMsg = e.toString(); + } finally { + if (!valid) { + s_logger.debug("ssvm http proxy " + _httpProxy + " is invalid: " + errMsg); + throw new ConfigurationException("ssvm http proxy " + _httpProxy + "is invalid: " + errMsg); + } + } + } if (s_logger.isInfoEnabled()) { s_logger.info("Secondary storage vm Manager is configured."); }