diff --git a/engine/storage/datamotion/src/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java b/engine/storage/datamotion/src/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java index c8a24c61874..96d1f5ab785 100644 --- a/engine/storage/datamotion/src/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java +++ b/engine/storage/datamotion/src/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java @@ -148,12 +148,6 @@ public class if (srcData.getType() == DataObjectType.TEMPLATE) { TemplateInfo template = (TemplateInfo)srcData; - if (template.getHypervisorType() == HypervisorType.Hyperv) { - if (s_logger.isDebugEnabled()) { - s_logger.debug("needCacheStorage false due to src TemplateInfo, which is DataObjectType.TEMPLATE of HypervisorType.Hyperv"); - } - return false; - } } if (s_logger.isDebugEnabled()) { @@ -304,11 +298,11 @@ public class Scope destScope = getZoneScope(destData.getDataStore().getScope()); DataStore cacheStore = cacheMgr.getCacheStorage(destScope); if (cacheStore == null) { - // need to find a nfs image store, assuming that can't copy volume + // need to find a nfs or cifs image store, assuming that can't copy volume // directly to s3 ImageStoreEntity imageStore = (ImageStoreEntity) this.dataStoreMgr.getImageStore(destScope.getScopeId()); - if (!imageStore.getProtocol().equalsIgnoreCase("nfs")) { - s_logger.debug("can't find a nfs image store"); + if (!imageStore.getProtocol().equalsIgnoreCase("nfs") && !imageStore.getProtocol().equalsIgnoreCase("cifs")) { + s_logger.debug("can't find a nfs (or cifs) image store to satisfy the need for a staging store"); return null; } diff --git a/engine/storage/src/org/apache/cloudstack/storage/datastore/protocol/DataStoreProtocol.java b/engine/storage/src/org/apache/cloudstack/storage/datastore/protocol/DataStoreProtocol.java index b27c96edbb7..ee9f0fbf7b5 100644 --- a/engine/storage/src/org/apache/cloudstack/storage/datastore/protocol/DataStoreProtocol.java +++ b/engine/storage/src/org/apache/cloudstack/storage/datastore/protocol/DataStoreProtocol.java @@ -17,7 +17,7 @@ package org.apache.cloudstack.storage.datastore.protocol; public enum DataStoreProtocol { - NFS("nfs"), ISCSI("iscsi"); + NFS("nfs"), CIFS("cifs"), ISCSI("iscsi"); private String name; diff --git a/plugins/storage/image/default/src/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackImageStoreLifeCycleImpl.java b/plugins/storage/image/default/src/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackImageStoreLifeCycleImpl.java index 21a5e0a7194..ee6f47b8fd1 100644 --- a/plugins/storage/image/default/src/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackImageStoreLifeCycleImpl.java +++ b/plugins/storage/image/default/src/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackImageStoreLifeCycleImpl.java @@ -16,14 +16,16 @@ // under the License. package org.apache.cloudstack.storage.datastore.lifecycle; -import com.cloud.agent.api.StoragePoolInfo; -import com.cloud.exception.InvalidParameterValueException; -import com.cloud.hypervisor.Hypervisor.HypervisorType; -import com.cloud.resource.Discoverer; -import com.cloud.resource.ResourceManager; -import com.cloud.storage.DataStoreRole; -import com.cloud.storage.ScopeType; -import com.cloud.utils.UriUtils; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.inject.Inject; + +import org.apache.log4j.Logger; + import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.HostScope; @@ -33,14 +35,15 @@ import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; import org.apache.cloudstack.storage.image.datastore.ImageStoreHelper; import org.apache.cloudstack.storage.image.datastore.ImageStoreProviderManager; import org.apache.cloudstack.storage.image.store.lifecycle.ImageStoreLifeCycle; -import org.apache.log4j.Logger; -import javax.inject.Inject; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import com.cloud.agent.api.StoragePoolInfo; +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.hypervisor.Hypervisor.HypervisorType; +import com.cloud.resource.Discoverer; +import com.cloud.resource.ResourceManager; +import com.cloud.storage.DataStoreRole; +import com.cloud.storage.ScopeType; +import com.cloud.utils.UriUtils; public class CloudStackImageStoreLifeCycleImpl implements ImageStoreLifeCycle { @@ -87,13 +90,21 @@ public class CloudStackImageStoreLifeCycleImpl implements ImageStoreLifeCycle { try { uri = new URI(UriUtils.encodeURIComponent(url)); if (uri.getScheme() == null) { - throw new InvalidParameterValueException("uri.scheme is null " + url + ", add nfs:// as a prefix"); + throw new InvalidParameterValueException("uri.scheme is null " + url + ", add nfs:// (or cifs://) as a prefix"); } else if (uri.getScheme().equalsIgnoreCase("nfs")) { if (uri.getHost() == null || uri.getHost().equalsIgnoreCase("") || uri.getPath() == null || uri.getPath().equalsIgnoreCase("")) { throw new InvalidParameterValueException( "Your host and/or path is wrong. Make sure it's of the format nfs://hostname/path"); } + } else if (uri.getScheme().equalsIgnoreCase("cifs")) { + // Don't validate against a URI encoded URI. + URI cifsUri = new URI(url); + String warnMsg = UriUtils.getCifsUriParametersProblems(cifsUri); + if (warnMsg != null) + { + throw new InvalidParameterValueException(warnMsg); + } } } catch (URISyntaxException e) { throw new InvalidParameterValueException(url + " is not a valid uri"); diff --git a/plugins/storage/volume/default/src/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java b/plugins/storage/volume/default/src/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java index b9b74243edd..9a7012494e4 100644 --- a/plugins/storage/volume/default/src/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java +++ b/plugins/storage/volume/default/src/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java @@ -143,13 +143,21 @@ public class CloudStackPrimaryDataStoreLifeCycleImpl implements PrimaryDataStore try { uri = new URI(UriUtils.encodeURIComponent(url)); if (uri.getScheme() == null) { - throw new InvalidParameterValueException("scheme is null " + url + ", add nfs:// as a prefix"); + throw new InvalidParameterValueException("scheme is null " + url + ", add nfs:// (or cifs://) as a prefix"); } else if (uri.getScheme().equalsIgnoreCase("nfs")) { String uriHost = uri.getHost(); String uriPath = uri.getPath(); if (uriHost == null || uriPath == null || uriHost.trim().isEmpty() || uriPath.trim().isEmpty()) { throw new InvalidParameterValueException("host or path is null, should be nfs://hostname/path"); } + } else if (uri.getScheme().equalsIgnoreCase("cifs")) { + // Don't validate against a URI encoded URI. + URI cifsUri = new URI(url); + String warnMsg = UriUtils.getCifsUriParametersProblems(cifsUri); + if (warnMsg != null) + { + throw new InvalidParameterValueException(warnMsg); + } } else if (uri.getScheme().equalsIgnoreCase("sharedMountPoint")) { String uriPath = uri.getPath(); if (uriPath == null) { @@ -193,6 +201,16 @@ public class CloudStackPrimaryDataStoreLifeCycleImpl implements PrimaryDataStore parameters.setHost(storageHost); parameters.setPort(port); parameters.setPath(hostPath); + } else if (scheme.equalsIgnoreCase("cifs")) { + if (port == -1) { + port = 445; + } + parameters.setType(StoragePoolType.NetworkFilesystem); + parameters.setHost(storageHost); + parameters.setPort(port); + parameters.setPath(hostPath); + parameters.setUserInfo(uri.getQuery()); + } else if (scheme.equalsIgnoreCase("file")) { if (port == -1) { port = 0; diff --git a/server/src/com/cloud/resource/ResourceManagerImpl.java b/server/src/com/cloud/resource/ResourceManagerImpl.java index 69cbd5bb0d2..76d7cf9d299 100755 --- a/server/src/com/cloud/resource/ResourceManagerImpl.java +++ b/server/src/com/cloud/resource/ResourceManagerImpl.java @@ -685,12 +685,20 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, try { uri = new URI(UriUtils.encodeURIComponent(url)); if (uri.getScheme() == null) { - throw new InvalidParameterValueException("uri.scheme is null " + url + ", add nfs:// as a prefix"); + throw new InvalidParameterValueException("uri.scheme is null " + url + ", add nfs:// (or cifs://) as a prefix"); } else if (uri.getScheme().equalsIgnoreCase("nfs")) { if (uri.getHost() == null || uri.getHost().equalsIgnoreCase("") || uri.getPath() == null || uri.getPath().equalsIgnoreCase("")) { throw new InvalidParameterValueException("Your host and/or path is wrong. Make sure it's of the format nfs://hostname/path"); } - } + } else if (uri.getScheme().equalsIgnoreCase("cifs")) { + // Don't validate against a URI encoded URI. + URI cifsUri = new URI(url); + String warnMsg = UriUtils.getCifsUriParametersProblems(cifsUri); + if (warnMsg != null) + { + throw new InvalidParameterValueException(warnMsg); + } + } } catch (URISyntaxException e) { throw new InvalidParameterValueException(url + " is not a valid uri"); } diff --git a/server/src/com/cloud/storage/StorageManagerImpl.java b/server/src/com/cloud/storage/StorageManagerImpl.java index b380692ff79..8417066cffb 100755 --- a/server/src/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/com/cloud/storage/StorageManagerImpl.java @@ -1406,11 +1406,19 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C try { uri = new URI(UriUtils.encodeURIComponent(newUrl)); if (uri.getScheme() == null) { - throw new InvalidParameterValueException("uri.scheme is null " + newUrl + ", add nfs:// as a prefix"); + throw new InvalidParameterValueException("uri.scheme is null " + newUrl + ", add nfs:// (or cifs://) as a prefix"); } else if (uri.getScheme().equalsIgnoreCase("nfs")) { if (uri.getHost() == null || uri.getHost().equalsIgnoreCase("") || uri.getPath() == null || uri.getPath().equalsIgnoreCase("")) { throw new InvalidParameterValueException("Your host and/or path is wrong. Make sure it's of the format nfs://hostname/path"); } + } else if (uri.getScheme().equalsIgnoreCase("cifs")) { + // Don't validate against a URI encoded URI. + URI cifsUri = new URI(newUrl); + String warnMsg = UriUtils.getCifsUriParametersProblems(cifsUri); + if (warnMsg != null) + { + throw new InvalidParameterValueException(warnMsg); + } } } catch (URISyntaxException e) { throw new InvalidParameterValueException(newUrl + " is not a valid uri"); diff --git a/server/src/com/cloud/test/DatabaseConfig.java b/server/src/com/cloud/test/DatabaseConfig.java index 63f77b66520..38a1abf7542 100755 --- a/server/src/com/cloud/test/DatabaseConfig.java +++ b/server/src/com/cloud/test/DatabaseConfig.java @@ -505,7 +505,7 @@ public class DatabaseConfig { stmt.setLong(14, 1238425896); boolean nfs = false; - if (url.startsWith("nfs")) { + if (url.startsWith("nfs") || url.startsWith("cifs")) { nfs = true; } if (nfs) { diff --git a/services/secondary-storage/conf/agent.properties b/services/secondary-storage/conf/agent.properties index aab82b63374..507ea4d0c4a 100644 --- a/services/secondary-storage/conf/agent.properties +++ b/services/secondary-storage/conf/agent.properties @@ -1,2 +1,4 @@ #mount.path=~/secondary-storage/ resource=org.apache.cloudstack.storage.resource.NfsSecondaryStorageResource +testCifsMount=cifs://192.168.1.1/CSHV3?user=administrator&password=1pass%40word1 +#testLocalRoot=test diff --git a/services/secondary-storage/conf/log4j.xml b/services/secondary-storage/conf/log4j.xml new file mode 100644 index 00000000000..9511f30aac2 --- /dev/null +++ b/services/secondary-storage/conf/log4j.xml @@ -0,0 +1,102 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/services/secondary-storage/pom.xml b/services/secondary-storage/pom.xml index ea4bfcaf477..9fe8da3299f 100644 --- a/services/secondary-storage/pom.xml +++ b/services/secondary-storage/pom.xml @@ -26,7 +26,10 @@ 4.3.0-SNAPSHOT ../pom.xml - + + true + + log4j log4j @@ -64,7 +67,10 @@ maven-surefire-plugin 2.14 - true + ${skipTests} + + file:${project.build.testSourceDirectory}/../conf/log4j.xml + diff --git a/services/secondary-storage/src/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResource.java b/services/secondary-storage/src/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResource.java index c55c2361bd5..926a1e8a85f 100644 --- a/services/secondary-storage/src/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResource.java +++ b/services/secondary-storage/src/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResource.java @@ -82,15 +82,8 @@ public class LocalNfsSecondaryStorageResource extends NfsSecondaryStorageResourc synchronized public String getRootDir(String secUrl) { try { URI uri = new URI(secUrl); - String nfsHost = uri.getHost(); - - InetAddress nfsHostAddr = InetAddress.getByName(nfsHost); - String nfsHostIp = nfsHostAddr.getHostAddress(); - String nfsPath = nfsHostIp + ":" + uri.getPath(); - String dir = UUID.nameUUIDFromBytes(nfsPath.getBytes()).toString(); - String root = _parent + "/" + dir; - mount(root, nfsPath); - return root; + String dir = mountUri(uri); + return _parent + "/" + dir; } catch (Exception e) { String msg = "GetRootDir for " + secUrl + " failed due to " + e.toString(); s_logger.error(msg, e); @@ -99,74 +92,30 @@ public class LocalNfsSecondaryStorageResource extends NfsSecondaryStorageResourc } @Override - protected String mount(String root, String nfsPath) { - File file = new File(root); - if (!file.exists()) { - if (_storage.mkdir(root)) { - s_logger.debug("create mount point: " + root); - } else { - s_logger.debug("Unable to create mount point: " + root); - return null; - } + protected void mount(String localRootPath, String remoteDevice, URI uri) { + ensureLocalRootPathExists(localRootPath, uri); + + if (mountExists(localRootPath, uri)) { + return; } - Script script = null; - String result = null; - script = new Script(!_inSystemVM, "mount", _timeout, s_logger); - List res = new ArrayList(); - ZfsPathParser parser = new ZfsPathParser(root); - script.execute(parser); - res.addAll(parser.getPaths()); - for (String s : res) { - if (s.contains(root)) { - s_logger.debug("mount point " + root + " already exists"); - return root; - } - } + attemptMount(localRootPath, remoteDevice, uri); - Script command = new Script(!_inSystemVM, "mount", _timeout, s_logger); - command.add("-t", "nfs"); - if ("Mac OS X".equalsIgnoreCase(System.getProperty("os.name"))) { - command.add("-o", "resvport"); - } - if (_inSystemVM) { - // Fedora Core 12 errors out with any -o option executed from java - command.add("-o", "soft,timeo=133,retrans=2147483647,tcp,acdirmax=0,acdirmin=0"); - } - command.add(nfsPath); - command.add(root); - result = command.execute(); + // Change permissions for the mountpoint - seems to bypass authentication + Script script = new Script(true, "chmod", _timeout, s_logger); + script.add("777", localRootPath); + String result = script.execute(); if (result != null) { - s_logger.warn("Unable to mount " + nfsPath + " due to " + result); - file = new File(root); - if (file.exists()) - file.delete(); - return null; + String errMsg = "Unable to set permissions for " + localRootPath + " due to " + result; + s_logger.error(errMsg); + throw new CloudRuntimeException(errMsg); } - s_logger.debug("Successfully mount " + nfsPath); - - // Change permissions for the mountpoint - script = new Script(true, "chmod", _timeout, s_logger); - script.add("777", root); - result = script.execute(); - if (result != null) { - s_logger.warn("Unable to set permissions for " + root + " due to " + result); - return null; - } - s_logger.debug("Successfully set 777 permission for " + root); + s_logger.debug("Successfully set 777 permission for " + localRootPath); // XXX: Adding the check for creation of snapshots dir here. Might have // to move it somewhere more logical later. - if (!checkForSnapshotsDir(root)) { - return null; - } - - // Create the volumes dir - if (!checkForVolumesDir(root)) { - return null; - } - - return root; + checkForSnapshotsDir(localRootPath); + checkForVolumesDir(localRootPath); } } diff --git a/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java b/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java index 3a563cb6655..3ef950b1a0b 100755 --- a/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java +++ b/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java @@ -27,6 +27,7 @@ import java.io.*; import java.math.BigInteger; import java.net.InetAddress; import java.net.URI; +import java.net.UnknownHostException; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.ArrayList; @@ -59,8 +60,10 @@ import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.lang.StringUtils; import org.apache.http.HttpEntity; import org.apache.http.HttpResponse; +import org.apache.http.NameValuePair; import org.apache.http.client.HttpClient; import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.utils.URLEncodedUtils; import org.apache.http.impl.client.DefaultHttpClient; import org.apache.log4j.Logger; @@ -156,6 +159,14 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S final private String _tmpltpp = "template.properties"; protected String createTemplateFromSnapshotXenScript; + public void setParentPath(String path) { + _parent = path; + } + + public String getMountingRoot() { + return _parent; + } + @Override public void disconnected() { } @@ -1205,16 +1216,10 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S String secUrl = cmd.getSecUrl(); try { URI uri = new URI(secUrl); - String nfsHost = uri.getHost(); - - InetAddress nfsHostAddr = InetAddress.getByName(nfsHost); - String nfsHostIp = nfsHostAddr.getHostAddress(); + String nfsHostIp = getUriHostIp(uri); addRouteToInternalIpOrCidr(_storageGateway, _storageIp, _storageNetmask, nfsHostIp); - String nfsPath = nfsHostIp + ":" + uri.getPath(); - String dir = UUID.nameUUIDFromBytes(nfsPath.getBytes()).toString(); - String root = _parent + "/" + dir; - mount(root, nfsPath); + String dir = mountUri(uri); configCerts(cmd.getCerts()); @@ -1893,15 +1898,8 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S } try { URI uri = new URI(secUrl); - String nfsHost = uri.getHost(); - - InetAddress nfsHostAddr = InetAddress.getByName(nfsHost); - String nfsHostIp = nfsHostAddr.getHostAddress(); - String nfsPath = nfsHostIp + ":" + uri.getPath(); - String dir = UUID.nameUUIDFromBytes(nfsPath.getBytes()).toString(); - String root = _parent + "/" + dir; - mount(root, nfsPath); - return root; + String dir = mountUri(uri); + return _parent + "/" + dir; } catch (Exception e) { String msg = "GetRootDir for " + secUrl + " failed due to " + e.toString(); s_logger.error(msg, e); @@ -1968,6 +1966,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S String inSystemVM = (String) params.get("secondary.storage.vm"); if (inSystemVM == null || "true".equalsIgnoreCase(inSystemVM)) { + s_logger.debug("conf secondary.storage.vm is true, act as if executing in SSVM"); _inSystemVM = true; } @@ -1984,24 +1983,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S _timeout = NumbersUtil.parseInt(value, 1440) * 1000; _storage = (StorageLayer) params.get(StorageLayer.InstanceConfigKey); - if (_storage == null) { - value = (String) params.get(StorageLayer.ClassConfigKey); - if (value == null) { - value = "com.cloud.storage.JavaStorageLayer"; - } - - try { - Class clazz = Class.forName(value); - _storage = (StorageLayer) clazz.newInstance(); - _storage.configure("StorageLayer", params); - } catch (ClassNotFoundException e) { - throw new ConfigurationException("Unable to find class " + value); - } catch (InstantiationException e) { - throw new ConfigurationException("Unable to find class " + value); - } catch (IllegalAccessException e) { - throw new ConfigurationException("Unable to find class " + value); - } - } + configureStorageLayerClass(params); if (_inSystemVM) { _storage.mkdirs(_parent); @@ -2090,6 +2072,28 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S return true; } + protected void configureStorageLayerClass(Map params) throws ConfigurationException { + String value; + if (_storage == null) { + value = (String) params.get(StorageLayer.ClassConfigKey); + if (value == null) { + value = "com.cloud.storage.JavaStorageLayer"; + } + + try { + Class clazz = Class.forName(value); + _storage = (StorageLayer) clazz.newInstance(); + _storage.configure("StorageLayer", params); + } catch (ClassNotFoundException e) { + throw new ConfigurationException("Unable to find class " + value); + } catch (InstantiationException e) { + throw new ConfigurationException("Unable to find class " + value); + } catch (IllegalAccessException e) { + throw new ConfigurationException("Unable to find class " + value); + } + } + } + private void startAdditionalServices() { if (!_inSystemVM) { return; @@ -2212,60 +2216,222 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S return result; } - protected String mount(String root, String nfsPath) { - File file = new File(root); - if (!file.exists()) { - if (_storage.mkdir(root)) { - s_logger.debug("create mount point: " + root); - } else { - s_logger.debug("Unable to create mount point: " + root); - return null; - } + /** + * Mount remote device named on local file system on subfolder of _parent + * field. + *

+ * + * Supported schemes are "nfs" and "cifs". + *

+ * + * CIFS parameters are documented with mount.cifs at + * http://linux.die.net/man/8/mount.cifs + * For simplicity, when a URI is used to specify a CIFS share, + * options such as domain,user,password are passed as query parameters. + * + * @param uri + * crresponding to the remote device. Will throw for unsupported + * scheme. + * @return name of folder in _parent that device was mounted. + * @throws UnknownHostException + */ + protected String mountUri(URI uri) throws UnknownHostException { + String uriHostIp = getUriHostIp(uri); + String nfsPath = uriHostIp + ":" + uri.getPath(); + + // Single means of calculating mount directory regardless of scheme + String dir = UUID.nameUUIDFromBytes(nfsPath.getBytes()).toString(); + String localRootPath = _parent + "/" + dir; + + // remote device syntax varies by scheme. + String remoteDevice; + if (uri.getScheme().equals("cifs")) { + remoteDevice = "//" + uriHostIp + uri.getPath(); + s_logger.debug("Mounting device with cifs-style path of " + + remoteDevice); + } + else + { + remoteDevice = nfsPath; + s_logger.debug("Mounting device with nfs-style path of " + + remoteDevice); } - Script script = null; - String result = null; - script = new Script(!_inSystemVM, "mount", _timeout, s_logger); - List res = new ArrayList(); - ZfsPathParser parser = new ZfsPathParser(root); - script.execute(parser); - res.addAll(parser.getPaths()); - for (String s : res) { - if (s.contains(root)) { - return root; - } + mount(localRootPath, remoteDevice, uri); + + return dir; + } + + + protected void umount(String localRootPath, URI uri) { + ensureLocalRootPathExists(localRootPath, uri); + + if (!mountExists(localRootPath, uri)) { + return; } Script command = new Script(!_inSystemVM, "mount", _timeout, s_logger); - command.add("-t", "nfs"); - if (_inSystemVM) { - // Fedora Core 12 errors out with any -o option executed from java - command.add("-o", "soft,timeo=133,retrans=2147483647,tcp,acdirmax=0,acdirmin=0"); - } - command.add(nfsPath); - command.add(root); - result = command.execute(); + command.add(localRootPath); + String result = command.execute(); if (result != null) { - s_logger.warn("Unable to mount " + nfsPath + " due to " + result); - file = new File(root); + // Fedora Core 12 errors out with any -o option executed from java + String errMsg = "Unable to umount " + localRootPath + " due to " + + result; + s_logger.error(errMsg); + File file = new File(localRootPath); if (file.exists()) { file.delete(); } - return null; + throw new CloudRuntimeException(errMsg); } + s_logger.debug("Successfully umounted " + localRootPath); + } + + protected void mount(String localRootPath, String remoteDevice, URI uri) { + s_logger.debug("mount " + uri.toString() + " on " + localRootPath); + ensureLocalRootPathExists(localRootPath, uri); + + if (mountExists(localRootPath, uri)) { + return; + } + + attemptMount(localRootPath, remoteDevice, uri); // XXX: Adding the check for creation of snapshots dir here. Might have // to move it somewhere more logical later. - if (!checkForSnapshotsDir(root)) { - return null; + checkForSnapshotsDir(localRootPath); + checkForVolumesDir(localRootPath); + } + + protected void attemptMount(String localRootPath, String remoteDevice, URI uri) { + String result; + s_logger.debug("Make cmdline call to mount " + remoteDevice + " at " + + localRootPath + " based on uri " + uri); + Script command = new Script(!_inSystemVM, "mount", _timeout, s_logger); + + String scheme = uri.getScheme().toLowerCase(); + command.add("-t", scheme); + + if (scheme.equals("nfs")) { + if ("Mac OS X".equalsIgnoreCase(System.getProperty("os.name"))) { + // See http://wiki.qnap.com/wiki/Mounting_an_NFS_share_from_OS_X + command.add("-o", "resvport"); + } + if (_inSystemVM) { + command.add("-o", + "soft,timeo=133,retrans=2147483647,tcp,acdirmax=0,acdirmin=0"); + } + } else if (scheme.equals("cifs")) { + String extraOpts = parseCifsMountOptions(uri); + + // nfs acdirmax / acdirmin correspoonds to CIFS actimeo (see + // http://linux.die.net/man/8/mount.cifs) + // no equivalent to nfs timeo, retrans or tcp in CIFS + // todo: allow security mode to be set. + command.add("-o", extraOpts + "soft,actimeo=0"); + } else { + String errMsg = "Unsupported storage device scheme " + scheme + + " in uri " + uri.toString(); + s_logger.error(errMsg); + throw new CloudRuntimeException(errMsg); } - // Create the volumes dir - if (!checkForVolumesDir(root)) { - return null; + command.add(remoteDevice); + command.add(localRootPath); + result = command.execute(); + if (result != null) { + // Fedora Core 12 errors out with any -o option executed from java + String errMsg = "Unable to mount " + remoteDevice + " at " + + localRootPath + " due to " + result; + s_logger.error(errMsg); + File file = new File(localRootPath); + if (file.exists()) { + file.delete(); + } + throw new CloudRuntimeException(errMsg); + } + s_logger.debug("Successfully mounted " + remoteDevice + " at " + localRootPath); + } + + protected String parseCifsMountOptions(URI uri) { + List args = URLEncodedUtils.parse(uri, "UTF-8"); + boolean foundUser = false; + boolean foundPswd = false; + String extraOpts = ""; + for (NameValuePair nvp : args) { + String name = nvp.getName(); + if (name.equals("user")) { + foundUser = true; + s_logger.debug("foundUser is" + foundUser); + } + else if (name.equals("password")) { + foundPswd = true; + s_logger.debug("foundPswd is" + foundPswd); + } + + extraOpts = extraOpts + name + "=" + nvp.getValue() + ","; } - return root; + if (s_logger.isDebugEnabled()) + { + s_logger.error("extraOpts now " + extraOpts); + } + + if (!foundUser || !foundPswd) { + String errMsg = "Missing user and password from URI. Make sure they" + + "are in the query string and separated by '&'. E.g. " + + "cifs://example.com/some_share?user=foo&password=bar"; + s_logger.error(errMsg); + throw new CloudRuntimeException(errMsg); + } + return extraOpts; + } + + protected boolean mountExists(String localRootPath, URI uri) { + Script script = null; + script = new Script(!_inSystemVM, "mount", _timeout, s_logger); + + List res = new ArrayList(); + ZfsPathParser parser = new ZfsPathParser(localRootPath); + script.execute(parser); + res.addAll(parser.getPaths()); + for (String s : res) { + if (s.contains(localRootPath)) { + s_logger.debug("Some device already mounted at " + + localRootPath + ", no need to mount " + + uri.toString()); + return true; + } + } + return false; + } + + protected void ensureLocalRootPathExists(String localRootPath, URI uri) { + s_logger.debug("making available " + localRootPath + " on " + uri.toString()); + File file = new File(localRootPath); + s_logger.debug("local folder for mount will be " + file.getPath()); + if (!file.exists()) { + s_logger.debug("create mount point: " + file.getPath()); + _storage.mkdir(file.getPath()); + + // Need to check after mkdir to allow O/S to complete operation + if (!file.exists()) { + String errMsg = "Unable to create local folder for: " + + localRootPath + + " in order to mount " + uri.toString(); + s_logger.error(errMsg); + throw new CloudRuntimeException(errMsg); + } + } + } + + protected String getUriHostIp(URI uri) throws UnknownHostException { + String nfsHost = uri.getHost(); + InetAddress nfsHostAddr = InetAddress.getByName(nfsHost); + String nfsHostIp = nfsHostAddr.getHostAddress(); + s_logger.info("Determined host " + nfsHost + " corresponds to IP " + + nfsHostIp); + return nfsHostIp; } @Override diff --git a/services/secondary-storage/src/org/apache/cloudstack/storage/resource/SecondaryStorageDiscoverer.java b/services/secondary-storage/src/org/apache/cloudstack/storage/resource/SecondaryStorageDiscoverer.java index 5d6d61f740a..0e5221b99d4 100755 --- a/services/secondary-storage/src/org/apache/cloudstack/storage/resource/SecondaryStorageDiscoverer.java +++ b/services/secondary-storage/src/org/apache/cloudstack/storage/resource/SecondaryStorageDiscoverer.java @@ -83,13 +83,16 @@ public class SecondaryStorageDiscoverer extends DiscovererBase implements Discov @Override public Map> find(long dcId, Long podId, Long clusterId, URI uri, String username, String password, List hostTags) { - if (!uri.getScheme().equalsIgnoreCase("nfs") && !uri.getScheme().equalsIgnoreCase("file") - && !uri.getScheme().equalsIgnoreCase("iso") && !uri.getScheme().equalsIgnoreCase("dummy")) { + if (!uri.getScheme().equalsIgnoreCase("nfs") && !uri.getScheme().equalsIgnoreCase("cifs") + && !uri.getScheme().equalsIgnoreCase("file") + && !uri.getScheme().equalsIgnoreCase("iso") + && !uri.getScheme().equalsIgnoreCase("dummy")) { s_logger.debug("It's not NFS or file or ISO, so not a secondary storage server: " + uri.toString()); return null; } - if (uri.getScheme().equalsIgnoreCase("nfs") || uri.getScheme().equalsIgnoreCase("iso")) { + if (uri.getScheme().equalsIgnoreCase("nfs") || uri.getScheme().equalsIgnoreCase("cifs") + || uri.getScheme().equalsIgnoreCase("iso")) { return createNfsSecondaryStorageResource(dcId, podId, uri); } else if (uri.getScheme().equalsIgnoreCase("file")) { return createLocalSecondaryStorageResource(dcId, podId, uri); diff --git a/services/secondary-storage/src/org/apache/cloudstack/storage/template/DownloadManagerImpl.java b/services/secondary-storage/src/org/apache/cloudstack/storage/template/DownloadManagerImpl.java index bf68b299c5c..20e39ee8f1e 100755 --- a/services/secondary-storage/src/org/apache/cloudstack/storage/template/DownloadManagerImpl.java +++ b/services/secondary-storage/src/org/apache/cloudstack/storage/template/DownloadManagerImpl.java @@ -566,7 +566,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager td = new LocalTemplateDownloader(_storage, url, tmpDir, maxTemplateSizeInBytes, new Completion(jobId)); } else if (uri.getScheme().equalsIgnoreCase("scp")) { td = new ScpTemplateDownloader(_storage, url, tmpDir, maxTemplateSizeInBytes, new Completion(jobId)); - } else if (uri.getScheme().equalsIgnoreCase("nfs")) { + } else if (uri.getScheme().equalsIgnoreCase("nfs") || uri.getScheme().equalsIgnoreCase("cifs")) { td = null; // TODO: implement this. throw new CloudRuntimeException("Scheme is not supported " + url); diff --git a/services/secondary-storage/test/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResourceTest.java b/services/secondary-storage/test/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResourceTest.java index 0c355ec884d..34d510da19a 100644 --- a/services/secondary-storage/test/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResourceTest.java +++ b/services/secondary-storage/test/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResourceTest.java @@ -18,7 +18,29 @@ */ package org.apache.cloudstack.storage.resource; -import com.cloud.agent.api.Answer; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.Map; +import java.util.Properties; +import java.util.UUID; + +import javax.naming.ConfigurationException; + +import junit.framework.Assert; +import junit.framework.TestCase; + +import org.apache.log4j.Logger; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; + +import org.apache.cloudstack.storage.command.CopyCmdAnswer; +import org.apache.cloudstack.storage.command.CopyCommand; +import org.apache.cloudstack.storage.command.DownloadCommand; +import org.apache.cloudstack.storage.to.TemplateObjectTO; + import com.cloud.agent.api.storage.DownloadAnswer; import com.cloud.agent.api.storage.ListTemplateAnswer; import com.cloud.agent.api.storage.ListTemplateCommand; @@ -27,25 +49,15 @@ import com.cloud.agent.api.to.NfsTO; import com.cloud.agent.api.to.SwiftTO; import com.cloud.storage.DataStoreRole; import com.cloud.storage.Storage; -import com.cloud.utils.SwiftUtil; -import junit.framework.Assert; -import junit.framework.TestCase; -import org.apache.cloudstack.api.command.user.tag.ListTagsCmd; -import org.apache.cloudstack.storage.command.CopyCmdAnswer; -import org.apache.cloudstack.storage.command.CopyCommand; -import org.apache.cloudstack.storage.command.DownloadCommand; -import org.apache.cloudstack.storage.to.SnapshotObjectTO; -import org.apache.cloudstack.storage.to.TemplateObjectTO; -import org.junit.Before; -import org.junit.Test; -import org.mockito.Mockito; - - -import javax.naming.ConfigurationException; -import java.util.HashMap; -import java.util.UUID; +import com.cloud.utils.PropertiesUtil; +import com.cloud.utils.exception.CloudRuntimeException; public class LocalNfsSecondaryStorageResourceTest extends TestCase { + private static Map testParams; + + private static final Logger s_logger = Logger + .getLogger(LocalNfsSecondaryStorageResourceTest.class.getName()); + LocalNfsSecondaryStorageResource resource; @Before @Override @@ -53,7 +65,15 @@ public class LocalNfsSecondaryStorageResourceTest extends TestCase { resource = new LocalNfsSecondaryStorageResource(); resource.setInSystemVM(true); + testParams = PropertiesUtil.toMap(loadProperties()); + resource.configureStorageLayerClass(testParams); + Object testLocalRoot = testParams.get("testLocalRoot"); resource.setParentPath("/mnt"); + + if (testLocalRoot != null) { + resource.setParentPath((String)testLocalRoot); + } + System.setProperty("paths.script", "/Users/edison/develop/asf-master/script"); //resource.configure("test", new HashMap()); } @@ -101,4 +121,25 @@ public class LocalNfsSecondaryStorageResourceTest extends TestCase { Assert.assertTrue(listAnswer.getTemplateInfo().size() > 0); } + public static Properties loadProperties() throws ConfigurationException { + Properties properties = new Properties(); + final File file = PropertiesUtil.findConfigFile("agent.properties"); + if (file == null) { + throw new ConfigurationException( + "Unable to find agent.properties."); + } + + s_logger.info("agent.properties found at " + file.getAbsolutePath()); + + try { + properties.load(new FileInputStream(file)); + } catch (final FileNotFoundException ex) { + throw new CloudRuntimeException("Cannot find the file: " + + file.getAbsolutePath(), ex); + } catch (final IOException ex) { + throw new CloudRuntimeException("IOException in reading " + + file.getAbsolutePath(), ex); + } + return properties; + } } diff --git a/services/secondary-storage/test/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResourceTest.java b/services/secondary-storage/test/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResourceTest.java new file mode 100644 index 00000000000..a805c78b822 --- /dev/null +++ b/services/secondary-storage/test/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResourceTest.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.resource; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.net.URI; +import java.util.Map; +import java.util.Properties; + +import javax.naming.ConfigurationException; + +import junit.framework.Assert; +import junit.framework.TestCase; + +import org.apache.log4j.Level; +import org.apache.log4j.Logger; +import org.junit.Before; +import org.junit.Test; + +import com.cloud.utils.PropertiesUtil; +import com.cloud.utils.exception.CloudRuntimeException; + +public class NfsSecondaryStorageResourceTest extends TestCase { + private static Map testParams; + + private static final Logger s_logger = Logger + .getLogger(NfsSecondaryStorageResourceTest.class.getName()); + + NfsSecondaryStorageResource resource; + @Before + @Override + public void setUp() throws ConfigurationException { + s_logger.setLevel(Level.ALL); + resource = new NfsSecondaryStorageResource(); + resource.setInSystemVM(true); + testParams = PropertiesUtil.toMap(loadProperties()); + resource.configureStorageLayerClass(testParams); + Object testLocalRoot = testParams.get("testLocalRoot"); + if (testLocalRoot != null) { + resource.setParentPath((String)testLocalRoot); + } + } + + @Test + public void testMount() throws Exception { + String sampleUriStr = "cifs://192.168.1.128/CSHV3?user=administrator&password=1pass%40word1&foo=bar"; + URI sampleUri = new URI(sampleUriStr); + + s_logger.info("Check HostIp parsing"); + String hostIpStr = resource.getUriHostIp(sampleUri); + Assert.assertEquals("Expected host IP " + sampleUri.getHost() + + " and actual host IP " + hostIpStr + " differ.", + sampleUri.getHost(), hostIpStr); + + s_logger.info("Check option parsing"); + String expected = "user=administrator,password=1pass@word1,foo=bar,"; + String actualOpts = resource.parseCifsMountOptions(sampleUri); + Assert.assertEquals("Options should be " + expected + " and not " + + actualOpts, expected, actualOpts); + + // attempt a configured mount + final Map params = + PropertiesUtil.toMap(loadProperties()); + String sampleMount = (String)params.get("testCifsMount"); + if ( !sampleMount.isEmpty()) + { + s_logger.info("functional test, mount " + sampleMount); + URI realMntUri = new URI(sampleMount); + String mntSubDir = resource.mountUri(realMntUri); + s_logger.info("functional test, umount " + mntSubDir); + resource.umount(resource.getMountingRoot() + mntSubDir, realMntUri); + } + else { + s_logger.info("no entry for testCifsMount in " + + "./conf/agent.properties - skip functional test"); + } + } + + public static Properties loadProperties() throws ConfigurationException { + Properties properties = new Properties(); + final File file = PropertiesUtil.findConfigFile("agent.properties"); + if (file == null) { + throw new ConfigurationException( + "Unable to find agent.properties."); + } + + s_logger.info("agent.properties found at " + file.getAbsolutePath()); + + try { + properties.load(new FileInputStream(file)); + } catch (final FileNotFoundException ex) { + throw new CloudRuntimeException("Cannot find the file: " + + file.getAbsolutePath(), ex); + } catch (final IOException ex) { + throw new CloudRuntimeException("IOException in reading " + + file.getAbsolutePath(), ex); + } + return properties; + } + +} diff --git a/utils/src/com/cloud/utils/UriUtils.java b/utils/src/com/cloud/utils/UriUtils.java index 1ff4d729cbf..4759710b95c 100644 --- a/utils/src/com/cloud/utils/UriUtils.java +++ b/utils/src/com/cloud/utils/UriUtils.java @@ -26,6 +26,7 @@ import java.net.URI; import java.net.URISyntaxException; import java.net.URLEncoder; import java.net.UnknownHostException; +import java.util.List; import javax.net.ssl.HttpsURLConnection; @@ -36,8 +37,8 @@ import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager; import org.apache.commons.httpclient.UsernamePasswordCredentials; import org.apache.commons.httpclient.auth.AuthScope; import org.apache.commons.httpclient.methods.GetMethod; -import org.apache.http.HttpResponse; -import org.apache.http.client.methods.HttpGet; +import org.apache.http.NameValuePair; +import org.apache.http.client.utils.URLEncodedUtils; import org.apache.log4j.Logger; import com.cloud.utils.exception.CloudRuntimeException; @@ -101,6 +102,47 @@ public class UriUtils { return url; } + public static String getCifsUriParametersProblems(URI uri) { + if (!UriUtils.hostAndPathPresent(uri)) { + String errMsg = "cifs URI missing host and/or path. " + + " Make sure it's of the format " + + "cifs://hostname/path?user=&password="; + s_logger.warn(errMsg); + return errMsg; + } + if (!UriUtils.cifsCredentialsPresent(uri)) + { + String errMsg = "cifs URI missing user and password details. " + + "Add them as query parameters, e.g. " + + "cifs://example.com/some_share?user=foo&password=bar"; + s_logger.warn(errMsg); + return errMsg; + } + return null; + } + + public static boolean hostAndPathPresent(URI uri) { + return !(uri.getHost() == null || uri.getHost().trim().isEmpty() + || uri.getPath() == null || uri.getPath().trim().isEmpty()); + } + + public static boolean cifsCredentialsPresent(URI uri) { + List args = URLEncodedUtils.parse(uri, "UTF-8"); + boolean foundUser = false; + boolean foundPswd = false; + for (NameValuePair nvp : args) { + String name = nvp.getName(); + if (name.equals("user")) { + foundUser = true; + s_logger.debug("foundUser is" + foundUser); + } + else if (name.equals("password")) { + foundPswd = true; + s_logger.debug("foundPswd is" + foundPswd); + } + } + return (foundUser && foundPswd); + } // Get the size of a file from URL response header. public static Long getRemoteSize(String url) { Long remoteSize = (long) 0;