diff --git a/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java b/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java index 7c4b8ca03d3..9782c2ec7c4 100755 --- a/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java +++ b/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java @@ -19,7 +19,6 @@ package org.apache.cloudstack.storage.resource; import static com.cloud.utils.S3Utils.mputFile; import static com.cloud.utils.S3Utils.putFile; import static com.cloud.utils.StringUtils.join; -import static com.cloud.utils.db.GlobalLock.executeWithNoWaitLock; import static java.lang.String.format; import static java.util.Arrays.asList; import static org.apache.commons.lang.StringUtils.substringAfterLast; @@ -44,7 +43,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; -import java.util.concurrent.Callable; import javax.naming.ConfigurationException; @@ -142,7 +140,6 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S private static final Logger s_logger = Logger.getLogger(NfsSecondaryStorageResource.class); private static final String TEMPLATE_ROOT_DIR = "template/tmpl"; - private static final String SNAPSHOT_ROOT_DIR = "snapshots"; private static final String VOLUME_ROOT_DIR = "volumes"; int _timeout; @@ -180,7 +177,6 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S private String _storageGateway; private final List nfsIps = new ArrayList(); protected String _parent = "/mnt/SecStorage"; - final private String _tmpltDir = "/var/cloudstack/template"; final private String _tmpltpp = "template.properties"; protected String createTemplateFromSnapshotXenScript; @@ -264,7 +260,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S String script = srcData.getObjectType() == DataObjectType.TEMPLATE ? createTmpltScr : createVolScr; int installTimeoutPerGig = 180 * 60 * 1000; - int imgSizeGigs = (int)Math.ceil(destFile.length() * 1.0d / (1024 * 1024 * 1024)); + long imgSizeGigs = (long)Math.ceil(destFile.length() * 1.0d / (1024 * 1024 * 1024)); imgSizeGigs++; // add one just in case long timeout = imgSizeGigs * installTimeoutPerGig; @@ -279,7 +275,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S String templateName = UUID.randomUUID().toString(); String templateFilename = templateName + "." + extension; Script scr = new Script(script, timeout, s_logger); - scr.add("-s", Integer.toString(imgSizeGigs)); // not used for now + scr.add("-s", Long.toString(imgSizeGigs)); // not used for now scr.add("-n", templateFilename); scr.add("-t", downloadPath); @@ -324,7 +320,9 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S try { String downloadPath = determineStorageTemplatePath(storagePath, destPath); final File downloadDirectory = _storage.getFile(downloadPath); - downloadDirectory.mkdirs(); + if (!downloadDirectory.mkdirs()) { + return new CopyCmdAnswer("Failed to create download directory " + downloadPath); + } File destFile = SwiftUtil.getObject(swiftTO, downloadDirectory, srcData.getPath()); return postProcessing(destFile, downloadPath, destPath, srcData, destData); } catch (Exception e) { @@ -341,9 +339,8 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S String downloadPath = determineStorageTemplatePath(storagePath, destPath); final File downloadDirectory = _storage.getFile(downloadPath); - downloadDirectory.mkdirs(); - if (!downloadDirectory.exists()) { + if (!downloadDirectory.mkdirs()) { final String errMsg = "Unable to create directory " + downloadPath + " to copy from S3 to cache."; s_logger.error(errMsg); return new CopyCmdAnswer(errMsg); @@ -372,7 +369,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S } protected Answer copySnapshotToTemplateFromNfsToNfsXenserver(CopyCommand cmd, SnapshotObjectTO srcData, NfsTO srcDataStore, TemplateObjectTO destData, - NfsTO destDataStore) { + NfsTO destDataStore) { String srcMountPoint = getRootDir(srcDataStore.getUrl()); String snapshotPath = srcData.getPath(); int index = snapshotPath.lastIndexOf("/"); @@ -488,6 +485,8 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S processor = new QCOW2Processor(); } else if (srcFormat == ImageFormat.RAW) { processor = new RawImageProcessor(); + } else { + throw new ConfigurationException("Unknown image format " + srcFormat.toString()); } Map params = new HashMap(); @@ -550,8 +549,8 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S } else if (destDataStore instanceof SwiftTO) { //create template on the same data store CopyCmdAnswer answer = - (CopyCmdAnswer)copySnapshotToTemplateFromNfsToNfs(cmd, (SnapshotObjectTO)srcData, (NfsTO)srcDataStore, (TemplateObjectTO)destData, - (NfsTO)srcDataStore); + (CopyCmdAnswer)copySnapshotToTemplateFromNfsToNfs(cmd, (SnapshotObjectTO)srcData, (NfsTO)srcDataStore, (TemplateObjectTO)destData, + (NfsTO)srcDataStore); if (!answer.getResult()) { return answer; } @@ -585,8 +584,8 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S } else if (destDataStore instanceof S3TO) { //create template on the same data store CopyCmdAnswer answer = - (CopyCmdAnswer)copySnapshotToTemplateFromNfsToNfs(cmd, (SnapshotObjectTO)srcData, (NfsTO)srcDataStore, (TemplateObjectTO)destData, - (NfsTO)srcDataStore); + (CopyCmdAnswer)copySnapshotToTemplateFromNfsToNfs(cmd, (SnapshotObjectTO)srcData, (NfsTO)srcDataStore, (TemplateObjectTO)destData, + (NfsTO)srcDataStore); if (!answer.getResult()) { return answer; } @@ -651,7 +650,6 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S return join(asList(TEMPLATE_ROOT_DIR, accountId, templateId, templateUniqueName), S3Utils.SEPARATOR); } - @SuppressWarnings("unchecked") private String determineS3TemplateNameFromKey(String key) { return StringUtils.substringAfterLast(StringUtils.substringBeforeLast(key, S3Utils.SEPARATOR), S3Utils.SEPARATOR); } @@ -661,12 +659,10 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S return join(asList(VOLUME_ROOT_DIR, accountId, volId), S3Utils.SEPARATOR); } - @SuppressWarnings("unchecked") protected Long determineS3VolumeIdFromKey(String key) { return Long.parseLong(StringUtils.substringAfterLast(StringUtils.substringBeforeLast(key, S3Utils.SEPARATOR), S3Utils.SEPARATOR)); } - @SuppressWarnings("unchecked") private String determineStorageTemplatePath(final String storagePath, String dataPath) { return join(asList(getRootDir(storagePath), dataPath), File.separator); } @@ -690,8 +686,8 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S _storage.mkdirs(filePath); } File destFile = new File(filePath + File.separator + name); - if (!destFile.exists()) { - destFile.createNewFile(); + if (!destFile.createNewFile()) { + s_logger.warn("Reusing existing file " + destFile.getPath()); } FileOutputStream outputStream = new FileOutputStream(destFile); entity.writeTo(outputStream); @@ -744,9 +740,9 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S } DownloadAnswer answer = - new DownloadAnswer(null, 100, null, VMTemplateStorageResourceAssoc.Status.DOWNLOADED, swiftPath, swiftPath, file.length(), file.length(), md5sum); + new DownloadAnswer(null, 100, null, VMTemplateStorageResourceAssoc.Status.DOWNLOADED, swiftPath, swiftPath, file.length(), file.length(), md5sum); return answer; - } catch (Exception e) { + } catch (IOException e) { s_logger.debug("Failed to register template into swift", e); return new DownloadAnswer(e.toString(), VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR); } finally { @@ -904,7 +900,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S Script command = new Script("/bin/bash", s_logger); command.add("-c"); command.add("/usr/bin/python /usr/local/cloud/systemvm/scripts/storage/secondary/swift -A " + swift.getUrl() + " -U " + swift.getAccount() + ":" + - swift.getUserName() + " -K " + swift.getKey() + " download " + container + " " + rfilename + " -o " + lFullPath); + swift.getUserName() + " -K " + swift.getKey() + " download " + container + " " + rfilename + " -o " + lFullPath); OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser(); String result = command.execute(parser); if (result != null) { @@ -916,7 +912,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S String[] lines = parser.getLines().split("\\n"); for (String line : lines) { if (line.contains("Errno") || line.contains("failed")) { - String errMsg = "swiftDownload failed , err=" + lines.toString(); + String errMsg = "swiftDownload failed , err=" + parser.getLines(); s_logger.warn(errMsg); return errMsg; } @@ -930,7 +926,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S Script command = new Script("/bin/bash", s_logger); command.add("-c"); command.add("cd " + ldir + ";/usr/bin/python /usr/local/cloud/systemvm/scripts/storage/secondary/swift -A " + swift.getUrl() + " -U " + swift.getAccount() + ":" + - swift.getUserName() + " -K " + swift.getKey() + " download " + container); + swift.getUserName() + " -K " + swift.getKey() + " download " + container); OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser(); String result = command.execute(parser); if (result != null) { @@ -942,7 +938,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S String[] lines = parser.getLines().split("\\n"); for (String line : lines) { if (line.contains("Errno") || line.contains("failed")) { - String errMsg = "swiftDownloadContainer failed , err=" + lines.toString(); + String errMsg = "swiftDownloadContainer failed , err=" + parser.getLines(); s_logger.warn(errMsg); return errMsg; } @@ -974,10 +970,10 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S command.add("-c"); if (size <= SWIFT_MAX_SIZE) { command.add("cd " + lDir + ";/usr/bin/python /usr/local/cloud/systemvm/scripts/storage/secondary/swift -A " + swift.getUrl() + " -U " + - swift.getAccount() + ":" + swift.getUserName() + " -K " + swift.getKey() + " upload " + container + " " + file); + swift.getAccount() + ":" + swift.getUserName() + " -K " + swift.getKey() + " upload " + container + " " + file); } else { command.add("cd " + lDir + ";/usr/bin/python /usr/local/cloud/systemvm/scripts/storage/secondary/swift -A " + swift.getUrl() + " -U " + - swift.getAccount() + ":" + swift.getUserName() + " -K " + swift.getKey() + " upload -S " + SWIFT_MAX_SIZE + " " + container + " " + file); + swift.getAccount() + ":" + swift.getUserName() + " -K " + swift.getKey() + " upload -S " + SWIFT_MAX_SIZE + " " + container + " " + file); } OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser(); String result = command.execute(parser); @@ -990,7 +986,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S String[] lines = parser.getLines().split("\\n"); for (String line : lines) { if (line.contains("Errno") || line.contains("failed")) { - String errMsg = "swiftUpload failed , err=" + lines.toString(); + String errMsg = "swiftUpload failed , err=" + parser.getLines(); s_logger.warn(errMsg); return errMsg; } @@ -1005,7 +1001,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S Script command = new Script("/bin/bash", s_logger); command.add("-c"); command.add("/usr/bin/python /usr/local/cloud/systemvm/scripts/storage/secondary/swift -A " + swift.getUrl() + " -U " + swift.getAccount() + ":" + - swift.getUserName() + " -K " + swift.getKey() + " list " + container + " " + rFilename); + swift.getUserName() + " -K " + swift.getKey() + " list " + container + " " + rFilename); OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser(); String result = command.execute(parser); if (result == null && parser.getLines() != null) { @@ -1027,7 +1023,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S Script command = new Script("/bin/bash", s_logger); command.add("-c"); command.add("/usr/bin/python /usr/local/cloud/systemvm/scripts/storage/secondary/swift -A " + swift.getUrl() + " -U " + swift.getAccount() + ":" + - swift.getUserName() + " -K " + swift.getKey() + " delete " + container + " " + object); + swift.getUserName() + " -K " + swift.getKey() + " delete " + container + " " + object); OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser(); String result = command.execute(parser); if (result != null) { @@ -1039,7 +1035,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S String[] lines = parser.getLines().split("\\n"); for (String line : lines) { if (line.contains("Errno") || line.contains("failed")) { - String errMsg = "swiftDelete failed , err=" + lines.toString(); + String errMsg = "swiftDelete failed , err=" + parser.getLines(); s_logger.warn(errMsg); return errMsg; } @@ -1094,7 +1090,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S return new Answer(cmd, true, String.format("Deleted snapshot %1%s from bucket %2$s.", path, bucket)); } catch (Exception e) { final String errorMessage = - String.format("Failed to delete snapshot %1$s from bucket %2$s due to the following error: %3$s", path, bucket, e.getMessage()); + String.format("Failed to delete snapshot %1$s from bucket %2$s due to the following error: %3$s", path, bucket, e.getMessage()); s_logger.error(errorMessage, e); return new Answer(cmd, false, errorMessage); } @@ -1123,21 +1119,6 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S } } - private String determineSnapshotS3Directory(final Long accountId, final Long volumeId) { - return join(S3Utils.SEPARATOR, SNAPSHOT_ROOT_DIR, accountId, volumeId); - } - - private String determineSnapshotS3Key(final Long accountId, final Long volumeId, final String snapshotFileName) { - - final String directoryName = determineSnapshotS3Directory(accountId, volumeId); - return join(S3Utils.SEPARATOR, directoryName, snapshotFileName); - - } - - private String determineSnapshotLocalDirectory(final String secondaryStorageUrl, final Long accountId, final Long volumeId) { - return join(File.pathSeparator, getRootDir(secondaryStorageUrl), SNAPSHOT_ROOT_DIR, accountId, volumeId); - } - private Answer execute(ComputeChecksumCommand cmd) { String relativeTemplatePath = cmd.getTemplatePath(); @@ -1207,7 +1188,6 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S } else { String prvKey = certs.getPrivKey(); String pubCert = certs.getPrivCert(); - String certChain = certs.getCertChain(); try { File prvKeyFile = File.createTempFile("prvkey", null); @@ -1265,69 +1245,6 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S } } - private String deleteSnapshotBackupFromLocalFileSystem(final String secondaryStorageUrl, final Long accountId, final Long volumeId, final String name, - final Boolean deleteAllFlag) { - String lPath = null; - int index = name.lastIndexOf(File.separator); - String snapshotPath = name.substring(0, index); - if (deleteAllFlag) { - lPath = getRootDir(secondaryStorageUrl) + File.separator + snapshotPath + File.separator + "*"; - } else { - lPath = getRootDir(secondaryStorageUrl) + File.separator + name + "*"; - } - - final String result = deleteLocalFile(lPath); - - if (result != null) { - return "failed to delete snapshot " + lPath + " , err=" + result; - } - - return null; - - } - - private String deleteSnapshotBackupfromS3(final S3TO s3, final Long accountId, final Long volumeId, final String name, final Boolean deleteAllFlag) { - - try { - - final String bucket = s3.getBucketName(); - - final String result = executeWithNoWaitLock(determineSnapshotLockId(accountId, volumeId), new Callable() { - - @Override - public String call() throws Exception { - - if (deleteAllFlag) { - S3Utils.deleteDirectory(s3, bucket, determineSnapshotS3Directory(accountId, volumeId)); - } else { - S3Utils.deleteObject(s3, bucket, determineSnapshotS3Key(accountId, volumeId, determineSnapshotBackupFilename(name))); - } - - return null; - - } - - }); - - return result; - - } catch (Exception e) { - - s_logger.error(String.format("Failed to delete snapshot backup for account id %1$s volume id %2$sfrom S3.", accountId, volumeId), e); - return e.getMessage(); - - } - - } - - private String determineSnapshotBackupFilename(final String snapshotUuid) { - return snapshotUuid + ".vhd"; - } - - private String determineSnapshotLockId(final Long accountId, final Long volumeId) { - return join("_", "SNAPSHOT", accountId, volumeId); - } - protected Answer deleteSnapshot(final DeleteCommand cmd) { DataTO obj = cmd.getData(); DataStoreTO dstore = obj.getDataStore(); @@ -1381,7 +1298,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S return new Answer(cmd, true, String.format("Deleted snapshot %1%s from bucket %2$s.", path, bucket)); } catch (Exception e) { final String errorMessage = - String.format("Failed to delete snapshot %1$s from bucket %2$s due to the following error: %3$s", path, bucket, e.getMessage()); + String.format("Failed to delete snapshot %1$s from bucket %2$s due to the following error: %3$s", path, bucket, e.getMessage()); s_logger.error(errorMessage, e); return new Answer(cmd, false, errorMessage); } @@ -1523,7 +1440,6 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S } DataStoreTO store = cmd.getDataStore(); if (store instanceof NfsTO) { - NfsTO nfs = (NfsTO)store; String root = getRootDir(cmd.getSecUrl()); Map templateInfos = _dlMgr.gatherVolumeInfo(root); return new ListVolumeAnswer(cmd.getSecUrl(), templateInfos); @@ -1571,45 +1487,6 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S } - private String setVhdParent(String lFullPath, String pFullPath) { - Script command = new Script("/bin/bash", s_logger); - command.add("-c"); - command.add("/bin/vhd-util modify -n " + lFullPath + " -p " + pFullPath); - String result = command.execute(); - if (result != null) { - String errMsg = "failed to set vhd parent, child " + lFullPath + " parent " + pFullPath + ", err=" + result; - s_logger.warn(errMsg); - return errMsg; - } - return null; - } - - private String createLocalDir(String folder) { - Script command = new Script("/bin/bash", s_logger); - command.add("-c"); - command.add("mkdir -p " + folder); - String result = command.execute(); - if (result != null) { - String errMsg = "Create local path " + folder + " failed , err=" + result; - s_logger.warn(errMsg); - return errMsg; - } - return null; - } - - private String deleteLocalDir(String folder) { - Script command = new Script("/bin/bash", s_logger); - command.add("-c"); - command.add("rmdir " + folder); - String result = command.execute(); - if (result != null) { - String errMsg = "Delete local path " + folder + " failed , err=" + result; - s_logger.warn(errMsg); - return errMsg; - } - return null; - } - private String deleteLocalFile(String fullPath) { Script command = new Script("/bin/bash", s_logger); command.add("-c"); @@ -1690,12 +1567,12 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S return new Answer(cmd, true, "Object with null install path does not exist on image store , no need to delete"); } switch (objType) { - case TEMPLATE: - return deleteTemplate(cmd); - case VOLUME: - return deleteVolume(cmd); - case SNAPSHOT: - return deleteSnapshot(cmd); + case TEMPLATE: + return deleteTemplate(cmd); + case VOLUME: + return deleteVolume(cmd); + case SNAPSHOT: + return deleteSnapshot(cmd); } return Answer.createUnsupportedCommandAnswer(cmd); } @@ -1777,7 +1654,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S return new Answer(cmd, true, String.format("Deleted template %1$s from bucket %2$s.", path, bucket)); } catch (Exception e) { final String errorMessage = - String.format("Failed to delete template %1$s from bucket %2$s due to the following error: %3$s", path, bucket, e.getMessage()); + String.format("Failed to delete template %1$s from bucket %2$s due to the following error: %3$s", path, bucket, e.getMessage()); s_logger.error(errorMessage, e); return new Answer(cmd, false, errorMessage); } @@ -2361,7 +2238,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S List args = URLEncodedUtils.parse(uri, "UTF-8"); boolean foundUser = false; boolean foundPswd = false; - String extraOpts = ""; + StringBuilder extraOpts = new StringBuilder(); for (NameValuePair nvp : args) { String name = nvp.getName(); if (name.equals("user")) { @@ -2369,10 +2246,10 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S s_logger.debug("foundUser is" + foundUser); } else if (name.equals("password")) { foundPswd = true; - s_logger.debug("foundPswd is" + foundPswd); + s_logger.debug("password is present in uri"); } - extraOpts = extraOpts + name + "=" + nvp.getValue() + ","; + extraOpts.append(name + "=" + nvp.getValue() + ","); } if (s_logger.isDebugEnabled()) { @@ -2381,12 +2258,12 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S if (!foundUser || !foundPswd) { String errMsg = - "Missing user and password from URI. Make sure they" + "are in the query string and separated by '&'. E.g. " - + "cifs://example.com/some_share?user=foo&password=bar"; + "Missing user and password from URI. Make sure they" + "are in the query string and separated by '&'. E.g. " + + "cifs://example.com/some_share?user=foo&password=bar"; s_logger.error(errMsg); throw new CloudRuntimeException(errMsg); } - return extraOpts; + return extraOpts.toString(); } protected boolean mountExists(String localRootPath, URI uri) { diff --git a/services/secondary-storage/src/org/apache/cloudstack/storage/resource/SecondaryStorageDiscoverer.java b/services/secondary-storage/src/org/apache/cloudstack/storage/resource/SecondaryStorageDiscoverer.java index fef034dbdbd..ee5064751be 100755 --- a/services/secondary-storage/src/org/apache/cloudstack/storage/resource/SecondaryStorageDiscoverer.java +++ b/services/secondary-storage/src/org/apache/cloudstack/storage/resource/SecondaryStorageDiscoverer.java @@ -32,12 +32,9 @@ import javax.naming.ConfigurationException; import org.apache.log4j.Logger; -import org.apache.cloudstack.framework.config.dao.ConfigurationDao; - import com.cloud.agent.AgentManager; import com.cloud.host.HostVO; import com.cloud.host.Status.Event; -import com.cloud.host.dao.HostDao; import com.cloud.hypervisor.Hypervisor; import com.cloud.resource.Discoverer; import com.cloud.resource.DiscovererBase; @@ -66,16 +63,12 @@ public class SecondaryStorageDiscoverer extends DiscovererBase implements Discov Random _random = new Random(System.currentTimeMillis()); @Inject - protected HostDao _hostDao = null; - @Inject protected VMTemplateDao _tmpltDao = null; @Inject protected VMTemplateZoneDao _vmTemplateZoneDao = null; @Inject protected VMTemplateDao _vmTemplateDao = null; @Inject - protected ConfigurationDao _configDao = null; - @Inject protected AgentManager _agentMgr = null; protected SecondaryStorageDiscoverer() { diff --git a/services/secondary-storage/src/org/apache/cloudstack/storage/template/DownloadManagerImpl.java b/services/secondary-storage/src/org/apache/cloudstack/storage/template/DownloadManagerImpl.java index 84daf27de7c..d45a6bbe607 100755 --- a/services/secondary-storage/src/org/apache/cloudstack/storage/template/DownloadManagerImpl.java +++ b/services/secondary-storage/src/org/apache/cloudstack/storage/template/DownloadManagerImpl.java @@ -62,6 +62,7 @@ import com.cloud.storage.VMTemplateStorageResourceAssoc; import com.cloud.storage.template.HttpTemplateDownloader; import com.cloud.storage.template.IsoProcessor; import com.cloud.storage.template.LocalTemplateDownloader; +import com.cloud.storage.template.OVAProcessor; import com.cloud.storage.template.Processor; import com.cloud.storage.template.Processor.FormatInfo; import com.cloud.storage.template.QCOW2Processor; @@ -75,7 +76,6 @@ import com.cloud.storage.template.TemplateDownloader.Status; import com.cloud.storage.template.TemplateLocation; import com.cloud.storage.template.TemplateProp; import com.cloud.storage.template.VhdProcessor; -import com.cloud.storage.template.OVAProcessor; import com.cloud.storage.template.VmdkProcessor; import com.cloud.utils.NumbersUtil; import com.cloud.utils.component.ManagerBase; @@ -104,14 +104,12 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager private static class DownloadJob { private final TemplateDownloader td; - private final String jobId; private final String tmpltName; private final boolean hvm; private final ImageFormat format; private String tmpltPath; private final String description; private String checksum; - private final Long accountId; private final String installPathPrefix; private long templatesize; private long templatePhysicalSize; @@ -122,11 +120,9 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager String installPathPrefix, ResourceType resourceType) { super(); this.td = td; - this.jobId = jobId; this.tmpltName = tmpltName; this.format = format; this.hvm = hvm; - this.accountId = accountId; description = descr; checksum = cksum; this.installPathPrefix = installPathPrefix; @@ -135,10 +131,6 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager this.resourceType = resourceType; } - public TemplateDownloader getTd() { - return td; - } - public String getDescription() { return description; } @@ -151,10 +143,6 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager return td; } - public String getJobId() { - return jobId; - } - public String getTmpltName() { return tmpltName; } @@ -167,10 +155,6 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager return hvm; } - public Long getAccountId() { - return accountId; - } - public long getId() { return id; } @@ -232,7 +216,6 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager private String _volumeDir; private String createTmpltScr; private String createVolScr; - private List processors; private ExecutorService threadPool; @@ -240,7 +223,6 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager private String listTmpltScr; private String listVolScr; private int installTimeoutPerGig = 180 * 60 * 1000; - private boolean _sslCopy; public void setThreadPool(ExecutorService threadPool) { this.threadPool = threadPool; @@ -268,55 +250,55 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager TemplateDownloader td = dj.getTemplateDownloader(); s_logger.info("Download Completion for jobId: " + jobId + ", status=" + status); s_logger.info("local: " + td.getDownloadLocalPath() + ", bytes=" + td.getDownloadedBytes() + ", error=" + td.getDownloadError() + ", pct=" + - td.getDownloadPercent()); + td.getDownloadPercent()); switch (status) { - case ABORTED: - case NOT_STARTED: - case UNRECOVERABLE_ERROR: - // TODO - dj.cleanup(); - break; - case UNKNOWN: - return; - case IN_PROGRESS: - s_logger.info("Resuming jobId: " + jobId + ", status=" + status); - td.setResume(true); - threadPool.execute(td); - break; - case RECOVERABLE_ERROR: - threadPool.execute(td); - break; - case DOWNLOAD_FINISHED: - if (!(td instanceof S3TemplateDownloader)) { - // we currently only create template.properties for NFS by - // running some post download script - td.setDownloadError("Download success, starting install "); - String result = postDownload(jobId); - if (result != null) { - s_logger.error("Failed post download script: " + result); - td.setStatus(Status.UNRECOVERABLE_ERROR); - td.setDownloadError("Failed post download script: " + result); - } else { - td.setStatus(Status.POST_DOWNLOAD_FINISHED); - td.setDownloadError("Install completed successfully at " + new SimpleDateFormat().format(new Date())); - } + case ABORTED: + case NOT_STARTED: + case UNRECOVERABLE_ERROR: + // TODO + dj.cleanup(); + break; + case UNKNOWN: + return; + case IN_PROGRESS: + s_logger.info("Resuming jobId: " + jobId + ", status=" + status); + td.setResume(true); + threadPool.execute(td); + break; + case RECOVERABLE_ERROR: + threadPool.execute(td); + break; + case DOWNLOAD_FINISHED: + if (!(td instanceof S3TemplateDownloader)) { + // we currently only create template.properties for NFS by + // running some post download script + td.setDownloadError("Download success, starting install "); + String result = postDownload(jobId); + if (result != null) { + s_logger.error("Failed post download script: " + result); + td.setStatus(Status.UNRECOVERABLE_ERROR); + td.setDownloadError("Failed post download script: " + result); } else { - // for s3 and swift, we skip post download step and just set - // status to trigger callback. td.setStatus(Status.POST_DOWNLOAD_FINISHED); - // set template size for S3 - S3TemplateDownloader std = (S3TemplateDownloader)td; - long size = std.totalBytes; - DownloadJob dnld = jobs.get(jobId); - dnld.setTemplatesize(size); - dnld.setTemplatePhysicalSize(size); - dnld.setTmpltPath(std.getDownloadLocalPath()); // update template path to include file name. + td.setDownloadError("Install completed successfully at " + new SimpleDateFormat().format(new Date())); } - dj.cleanup(); - break; - default: - break; + } else { + // for s3 and swift, we skip post download step and just set + // status to trigger callback. + td.setStatus(Status.POST_DOWNLOAD_FINISHED); + // set template size for S3 + S3TemplateDownloader std = (S3TemplateDownloader)td; + long size = std.totalBytes; + DownloadJob dnld = jobs.get(jobId); + dnld.setTemplatesize(size); + dnld.setTemplatePhysicalSize(size); + dnld.setTmpltPath(std.getDownloadLocalPath()); // update template path to include file name. + } + dj.cleanup(); + break; + default: + break; } } @@ -360,28 +342,12 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager DownloadJob dnld = jobs.get(jobId); TemplateDownloader td = dnld.getTemplateDownloader(); String resourcePath = dnld.getInstallPathPrefix(); // path with mount - // directory + // directory String finalResourcePath = dnld.getTmpltPath(); // template download - // path on secondary - // storage + // path on secondary + // storage ResourceType resourceType = dnld.getResourceType(); - /* - // once template path is set, remove the parent dir so that the template - // is installed with a relative path - String finalResourcePath = ""; - if (resourceType == ResourceType.TEMPLATE) { - finalResourcePath += _templateDir + File.separator + dnld.getAccountId() + File.separator + dnld.getId() + File.separator; - resourcePath = dnld.getInstallPathPrefix() + dnld.getAccountId() + File.separator + dnld.getId() + File.separator;// dnld.getTmpltName(); - } else { - finalResourcePath += _volumeDir + File.separator + dnld.getId() + File.separator; - resourcePath = dnld.getInstallPathPrefix() + dnld.getId() + File.separator;// dnld.getTmpltName(); - } - - _storage.mkdirs(resourcePath); - dnld.setTmpltPath(finalResourcePath); - */ - File originalTemplate = new File(td.getDownloadLocalPath()); String checkSum = computeCheckSum(originalTemplate); if (checkSum == null) { @@ -421,7 +387,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager scr.add("-t", resourcePath); scr.add("-f", td.getDownloadLocalPath()); // this is the temporary - // template file downloaded + // template file downloaded if (dnld.getChecksum() != null && dnld.getChecksum().length() > 1) { scr.add("-c", dnld.getChecksum()); } @@ -497,7 +463,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager @Override public String downloadS3Template(S3TO s3, long id, String url, String name, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum, - String installPathPrefix, String user, String password, long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType) { + String installPathPrefix, String user, String password, long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType) { UUID uuid = UUID.randomUUID(); String jobId = uuid.toString(); @@ -527,7 +493,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager @Override public String downloadPublicTemplate(long id, String url, String name, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum, - String installPathPrefix, String templatePath, String user, String password, long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType) { + String installPathPrefix, String templatePath, String user, String password, long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType) { UUID uuid = UUID.randomUUID(); String jobId = uuid.toString(); String tmpDir = installPathPrefix; @@ -540,52 +506,52 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager } // TO DO - define constant for volume properties. File file = - ResourceType.TEMPLATE == resourceType ? _storage.getFile(tmpDir + File.separator + TemplateLocation.Filename) : _storage.getFile(tmpDir + File.separator + - "volume.properties"); - if (file.exists()) { - file.delete(); - } + ResourceType.TEMPLATE == resourceType ? _storage.getFile(tmpDir + File.separator + TemplateLocation.Filename) : _storage.getFile(tmpDir + File.separator + + "volume.properties"); + if (file.exists()) { + file.delete(); + } - if (!file.createNewFile()) { - s_logger.warn("Unable to create new file: " + file.getAbsolutePath()); - return "Unable to create new file: " + file.getAbsolutePath(); - } + if (!file.createNewFile()) { + s_logger.warn("Unable to create new file: " + file.getAbsolutePath()); + return "Unable to create new file: " + file.getAbsolutePath(); + } - URI uri; - try { - uri = new URI(url); - } catch (URISyntaxException e) { - throw new CloudRuntimeException("URI is incorrect: " + url); - } - TemplateDownloader td; - if ((uri != null) && (uri.getScheme() != null)) { - if (uri.getScheme().equalsIgnoreCase("http") || uri.getScheme().equalsIgnoreCase("https")) { - td = new HttpTemplateDownloader(_storage, url, tmpDir, new Completion(jobId), maxTemplateSizeInBytes, user, password, proxy, resourceType); - } else if (uri.getScheme().equalsIgnoreCase("file")) { - td = new LocalTemplateDownloader(_storage, url, tmpDir, maxTemplateSizeInBytes, new Completion(jobId)); - } else if (uri.getScheme().equalsIgnoreCase("scp")) { - td = new ScpTemplateDownloader(_storage, url, tmpDir, maxTemplateSizeInBytes, new Completion(jobId)); - } else if (uri.getScheme().equalsIgnoreCase("nfs") || uri.getScheme().equalsIgnoreCase("cifs")) { - td = null; - // TODO: implement this. - throw new CloudRuntimeException("Scheme is not supported " + url); - } else { - throw new CloudRuntimeException("Scheme is not supported " + url); - } - } else { - throw new CloudRuntimeException("Unable to download from URL: " + url); - } - // NOTE the difference between installPathPrefix and templatePath - // here. instalPathPrefix is the absolute path for template - // including mount directory - // on ssvm, while templatePath is the final relative path on - // secondary storage. - DownloadJob dj = new DownloadJob(td, jobId, id, name, format, hvm, accountId, descr, cksum, installPathPrefix, resourceType); - dj.setTmpltPath(templatePath); - jobs.put(jobId, dj); - threadPool.execute(td); + URI uri; + try { + uri = new URI(url); + } catch (URISyntaxException e) { + throw new CloudRuntimeException("URI is incorrect: " + url); + } + TemplateDownloader td; + if ((uri != null) && (uri.getScheme() != null)) { + if (uri.getScheme().equalsIgnoreCase("http") || uri.getScheme().equalsIgnoreCase("https")) { + td = new HttpTemplateDownloader(_storage, url, tmpDir, new Completion(jobId), maxTemplateSizeInBytes, user, password, proxy, resourceType); + } else if (uri.getScheme().equalsIgnoreCase("file")) { + td = new LocalTemplateDownloader(_storage, url, tmpDir, maxTemplateSizeInBytes, new Completion(jobId)); + } else if (uri.getScheme().equalsIgnoreCase("scp")) { + td = new ScpTemplateDownloader(_storage, url, tmpDir, maxTemplateSizeInBytes, new Completion(jobId)); + } else if (uri.getScheme().equalsIgnoreCase("nfs") || uri.getScheme().equalsIgnoreCase("cifs")) { + td = null; + // TODO: implement this. + throw new CloudRuntimeException("Scheme is not supported " + url); + } else { + throw new CloudRuntimeException("Scheme is not supported " + url); + } + } else { + throw new CloudRuntimeException("Unable to download from URL: " + url); + } + // NOTE the difference between installPathPrefix and templatePath + // here. instalPathPrefix is the absolute path for template + // including mount directory + // on ssvm, while templatePath is the final relative path on + // secondary storage. + DownloadJob dj = new DownloadJob(td, jobId, id, name, format, hvm, accountId, descr, cksum, installPathPrefix, resourceType); + dj.setTmpltPath(templatePath); + jobs.put(jobId, dj); + threadPool.execute(td); - return jobId; + return jobId; } catch (IOException e) { s_logger.warn("Unable to download to " + tmpDir, e); return null; @@ -645,24 +611,24 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager public static VMTemplateHostVO.Status convertStatus(Status tds) { switch (tds) { - case ABORTED: - return VMTemplateHostVO.Status.NOT_DOWNLOADED; - case DOWNLOAD_FINISHED: - return VMTemplateHostVO.Status.DOWNLOAD_IN_PROGRESS; - case IN_PROGRESS: - return VMTemplateHostVO.Status.DOWNLOAD_IN_PROGRESS; - case NOT_STARTED: - return VMTemplateHostVO.Status.NOT_DOWNLOADED; - case RECOVERABLE_ERROR: - return VMTemplateHostVO.Status.NOT_DOWNLOADED; - case UNKNOWN: - return VMTemplateHostVO.Status.UNKNOWN; - case UNRECOVERABLE_ERROR: - return VMTemplateHostVO.Status.DOWNLOAD_ERROR; - case POST_DOWNLOAD_FINISHED: - return VMTemplateHostVO.Status.DOWNLOADED; - default: - return VMTemplateHostVO.Status.UNKNOWN; + case ABORTED: + return VMTemplateHostVO.Status.NOT_DOWNLOADED; + case DOWNLOAD_FINISHED: + return VMTemplateHostVO.Status.DOWNLOAD_IN_PROGRESS; + case IN_PROGRESS: + return VMTemplateHostVO.Status.DOWNLOAD_IN_PROGRESS; + case NOT_STARTED: + return VMTemplateHostVO.Status.NOT_DOWNLOADED; + case RECOVERABLE_ERROR: + return VMTemplateHostVO.Status.NOT_DOWNLOADED; + case UNKNOWN: + return VMTemplateHostVO.Status.UNKNOWN; + case UNRECOVERABLE_ERROR: + return VMTemplateHostVO.Status.DOWNLOAD_ERROR; + case POST_DOWNLOAD_FINISHED: + return VMTemplateHostVO.Status.DOWNLOADED; + default: + return VMTemplateHostVO.Status.UNKNOWN; } } @@ -680,7 +646,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager if (cmd.getUrl() == null) { return new DownloadAnswer(resourceType.toString() + " is corrupted on storage due to an invalid url , cannot download", - VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR); + VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR); } if (cmd.getName() == null) { @@ -697,27 +663,27 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager String password = null; if (cmd.getAuth() != null) { user = cmd.getAuth().getUserName(); - password = new String(cmd.getAuth().getPassword()); + password = cmd.getAuth().getPassword(); } // TO DO - Define Volume max size as well long maxDownloadSizeInBytes = - (cmd.getMaxDownloadSizeInBytes() == null) ? TemplateDownloader.DEFAULT_MAX_TEMPLATE_SIZE_IN_BYTES : (cmd.getMaxDownloadSizeInBytes()); + (cmd.getMaxDownloadSizeInBytes() == null) ? TemplateDownloader.DEFAULT_MAX_TEMPLATE_SIZE_IN_BYTES : (cmd.getMaxDownloadSizeInBytes()); String jobId = null; if (dstore instanceof S3TO) { jobId = - downloadS3Template((S3TO)dstore, cmd.getId(), cmd.getUrl(), cmd.getName(), cmd.getFormat(), cmd.isHvm(), cmd.getAccountId(), cmd.getDescription(), - cmd.getChecksum(), installPathPrefix, user, password, maxDownloadSizeInBytes, cmd.getProxy(), resourceType); + downloadS3Template((S3TO)dstore, cmd.getId(), cmd.getUrl(), cmd.getName(), cmd.getFormat(), cmd.isHvm(), cmd.getAccountId(), cmd.getDescription(), + cmd.getChecksum(), installPathPrefix, user, password, maxDownloadSizeInBytes, cmd.getProxy(), resourceType); } else { jobId = - downloadPublicTemplate(cmd.getId(), cmd.getUrl(), cmd.getName(), cmd.getFormat(), cmd.isHvm(), cmd.getAccountId(), cmd.getDescription(), - cmd.getChecksum(), installPathPrefix, cmd.getInstallPath(), user, password, maxDownloadSizeInBytes, cmd.getProxy(), resourceType); + downloadPublicTemplate(cmd.getId(), cmd.getUrl(), cmd.getName(), cmd.getFormat(), cmd.isHvm(), cmd.getAccountId(), cmd.getDescription(), + cmd.getChecksum(), installPathPrefix, cmd.getInstallPath(), user, password, maxDownloadSizeInBytes, cmd.getProxy(), resourceType); } sleep(); if (jobId == null) { return new DownloadAnswer("Internal Error", VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR); } return new DownloadAnswer(jobId, getDownloadPct(jobId), getDownloadError(jobId), getDownloadStatus2(jobId), getDownloadLocalPath(jobId), getInstallPath(jobId), - getDownloadTemplateSize(jobId), getDownloadTemplateSize(jobId), getDownloadCheckSum(jobId)); + getDownloadTemplateSize(jobId), getDownloadTemplateSize(jobId), getDownloadCheckSum(jobId)); } private void sleep() { @@ -745,29 +711,29 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager } TemplateDownloader td = dj.getTemplateDownloader(); switch (cmd.getRequest()) { - case GET_STATUS: - break; - case ABORT: - td.stopDownload(); - sleep(); - break; - case RESTART: - td.stopDownload(); - sleep(); - threadPool.execute(td); - break; - case PURGE: - td.stopDownload(); - answer = + case GET_STATUS: + break; + case ABORT: + td.stopDownload(); + sleep(); + break; + case RESTART: + td.stopDownload(); + sleep(); + threadPool.execute(td); + break; + case PURGE: + td.stopDownload(); + answer = new DownloadAnswer(jobId, getDownloadPct(jobId), getDownloadError(jobId), getDownloadStatus2(jobId), getDownloadLocalPath(jobId), - getInstallPath(jobId), getDownloadTemplateSize(jobId), getDownloadTemplatePhysicalSize(jobId), getDownloadCheckSum(jobId)); - jobs.remove(jobId); - return answer; - default: - break; // TODO + getInstallPath(jobId), getDownloadTemplateSize(jobId), getDownloadTemplatePhysicalSize(jobId), getDownloadCheckSum(jobId)); + jobs.remove(jobId); + return answer; + default: + break; // TODO } return new DownloadAnswer(jobId, getDownloadPct(jobId), getDownloadError(jobId), getDownloadStatus2(jobId), getDownloadLocalPath(jobId), getInstallPath(jobId), - getDownloadTemplateSize(jobId), getDownloadTemplatePhysicalSize(jobId), getDownloadCheckSum(jobId)); + getDownloadTemplateSize(jobId), getDownloadTemplatePhysicalSize(jobId), getDownloadCheckSum(jobId)); } private String getInstallPath(String jobId) { @@ -778,16 +744,6 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager return null; } - private String createTempDir(File rootDir, String name) throws IOException { - - File f = File.createTempFile(name, "", rootDir); - f.delete(); - f.mkdir(); - _storage.setWorldReadableAndWriteable(f); - return f.getAbsolutePath(); - - } - private List listVolumes(String rootdir) { List result = new ArrayList(); @@ -915,27 +871,6 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager return result; } - private int deleteDownloadDirectories(File downloadPath, int deleted) { - try { - if (downloadPath.exists()) { - File[] files = downloadPath.listFiles(); - for (int i = 0; i < files.length; i++) { - if (files[i].isDirectory()) { - deleteDownloadDirectories(files[i], deleted); - files[i].delete(); - deleted++; - } else { - files[i].delete(); - deleted++; - } - } - } - } catch (Exception ex) { - s_logger.info("Failed to clean up template downloads directory " + ex.toString()); - } - return deleted; - } - public static class ZfsPathParser extends OutputInterpreter { String _parent; List paths = new ArrayList(); @@ -992,11 +927,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager throw new ConfigurationException("Unable to instantiate " + value); } } - String useSsl = (String)params.get("sslcopy"); - if (useSsl != null) { - _sslCopy = Boolean.parseBoolean(useSsl); - } String inSystemVM = (String)params.get("secondary.storage.vm"); if (inSystemVM != null && "true".equalsIgnoreCase(inSystemVM)) { s_logger.info("DownloadManager: starting additional services since we are inside system vm"); @@ -1081,7 +1012,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager String intf = "eth1"; command.add("-c"); command.add("iptables -A OUTPUT -o " + intf + " -p tcp -m state --state NEW -m tcp --dport " + "80" + " -j REJECT;" + "iptables -A OUTPUT -o " + intf + - " -p tcp -m state --state NEW -m tcp --dport " + "443" + " -j REJECT;"); + " -p tcp -m state --state NEW -m tcp --dport " + "443" + " -j REJECT;"); String result = command.execute(); if (result != null) { @@ -1120,7 +1051,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager command = new Script("/bin/bash", s_logger); command.add("-c"); command.add("iptables -I INPUT -i " + intf + " -p tcp -m state --state NEW -m tcp --dport " + port + " -j ACCEPT;" + "iptables -I INPUT -i " + intf + - " -p tcp -m state --state NEW -m tcp --dport " + "443" + " -j ACCEPT;"); + " -p tcp -m state --state NEW -m tcp --dport " + "443" + " -j ACCEPT;"); result = command.execute(); if (result != null) { diff --git a/services/secondary-storage/src/org/apache/cloudstack/storage/template/UploadManagerImpl.java b/services/secondary-storage/src/org/apache/cloudstack/storage/template/UploadManagerImpl.java index dc0ef06b8ac..cdbc52dec5a 100755 --- a/services/secondary-storage/src/org/apache/cloudstack/storage/template/UploadManagerImpl.java +++ b/services/secondary-storage/src/org/apache/cloudstack/storage/template/UploadManagerImpl.java @@ -21,7 +21,6 @@ import java.net.URI; import java.net.URISyntaxException; import java.text.SimpleDateFormat; import java.util.Date; -import java.util.List; import java.util.Map; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; @@ -46,7 +45,6 @@ import com.cloud.storage.StorageLayer; import com.cloud.storage.Upload; import com.cloud.storage.UploadVO; import com.cloud.storage.template.FtpTemplateUploader; -import com.cloud.storage.template.Processor; import com.cloud.storage.template.TemplateUploader; import com.cloud.storage.template.TemplateUploader.Status; import com.cloud.storage.template.TemplateUploader.UploadCompleteCallback; @@ -72,87 +70,17 @@ public class UploadManagerImpl extends ManagerBase implements UploadManager { private static class UploadJob { private final TemplateUploader tu; - private final String jobId; - private final String name; - private final ImageFormat format; - private String tmpltPath; - private String description; - private String checksum; - private Long accountId; - private String installPathPrefix; - private long templatesize; - private long id; public UploadJob(TemplateUploader tu, String jobId, long id, String name, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum, String installPathPrefix) { super(); this.tu = tu; - this.jobId = jobId; - this.name = name; - this.format = format; - this.accountId = accountId; - this.description = descr; - this.checksum = cksum; - this.installPathPrefix = installPathPrefix; - this.templatesize = 0; - this.id = id; - } - - public TemplateUploader getTd() { - return tu; - } - - public String getDescription() { - return description; - } - - public String getChecksum() { - return checksum; - } - - public UploadJob(TemplateUploader td, String jobId, UploadCommand cmd) { - this.tu = td; - this.jobId = jobId; - this.name = cmd.getName(); - this.format = cmd.getFormat(); } public TemplateUploader getTemplateUploader() { return tu; } - public String getJobId() { - return jobId; - } - - public String getTmpltName() { - return name; - } - - public ImageFormat getFormat() { - return format; - } - - public Long getAccountId() { - return accountId; - } - - public long getId() { - return id; - } - - public void setTmpltPath(String tmpltPath) { - this.tmpltPath = tmpltPath; - } - - public String getTmpltPath() { - return tmpltPath; - } - - public String getInstallPathPrefix() { - return installPathPrefix; - } - public void cleanup() { if (tu != null) { String upldPath = tu.getUploadLocalPath(); @@ -163,30 +91,19 @@ public class UploadManagerImpl extends ManagerBase implements UploadManager { } } - public void setTemplatesize(long templatesize) { - this.templatesize = templatesize; - } - - public long getTemplatesize() { - return templatesize; - } } public static final Logger s_logger = Logger.getLogger(UploadManagerImpl.class); private ExecutorService threadPool; private final Map jobs = new ConcurrentHashMap(); private String parentDir; - private List _processors; - private String publicTemplateRepo; private final String extractMountPoint = "/mnt/SecStorage/extractmnt"; private StorageLayer _storage; - private int installTimeoutPerGig; - private boolean _sslCopy; private boolean hvm; @Override public String uploadPublicTemplate(long id, String url, String name, ImageFormat format, Long accountId, String descr, String cksum, String installPathPrefix, - String userName, String passwd, long templateSizeInBytes) { + String userName, String passwd, long templateSizeInBytes) { UUID uuid = UUID.randomUUID(); String jobId = uuid.toString(); @@ -253,24 +170,24 @@ public class UploadManagerImpl extends ManagerBase implements UploadManager { public static UploadVO.Status convertStatus(Status tds) { switch (tds) { - case ABORTED: - return UploadVO.Status.NOT_UPLOADED; - case UPLOAD_FINISHED: - return UploadVO.Status.UPLOAD_IN_PROGRESS; - case IN_PROGRESS: - return UploadVO.Status.UPLOAD_IN_PROGRESS; - case NOT_STARTED: - return UploadVO.Status.NOT_UPLOADED; - case RECOVERABLE_ERROR: - return UploadVO.Status.NOT_UPLOADED; - case UNKNOWN: - return UploadVO.Status.UNKNOWN; - case UNRECOVERABLE_ERROR: - return UploadVO.Status.UPLOAD_ERROR; - case POST_UPLOAD_FINISHED: - return UploadVO.Status.UPLOADED; - default: - return UploadVO.Status.UNKNOWN; + case ABORTED: + return UploadVO.Status.NOT_UPLOADED; + case UPLOAD_FINISHED: + return UploadVO.Status.UPLOAD_IN_PROGRESS; + case IN_PROGRESS: + return UploadVO.Status.UPLOAD_IN_PROGRESS; + case NOT_STARTED: + return UploadVO.Status.NOT_UPLOADED; + case RECOVERABLE_ERROR: + return UploadVO.Status.NOT_UPLOADED; + case UNKNOWN: + return UploadVO.Status.UNKNOWN; + case UNRECOVERABLE_ERROR: + return UploadVO.Status.UPLOAD_ERROR; + case POST_UPLOAD_FINISHED: + return UploadVO.Status.UPLOADED; + default: + return UploadVO.Status.UNKNOWN; } } @@ -296,29 +213,29 @@ public class UploadManagerImpl extends ManagerBase implements UploadManager { } TemplateUploader td = uj.getTemplateUploader(); switch (cmd.getRequest()) { - case GET_STATUS: - break; - case ABORT: - td.stopUpload(); - sleep(); - break; + case GET_STATUS: + break; + case ABORT: + td.stopUpload(); + sleep(); + break; /*case RESTART: td.stopUpload(); sleep(); threadPool.execute(td); break;*/ - case PURGE: - td.stopUpload(); - answer = + case PURGE: + td.stopUpload(); + answer = new UploadAnswer(jobId, getUploadPct(jobId), getUploadError(jobId), getUploadStatus2(jobId), getUploadLocalPath(jobId), getInstallPath(jobId), - getUploadTemplateSize(jobId)); - jobs.remove(jobId); - return answer; - default: - break; // TODO + getUploadTemplateSize(jobId)); + jobs.remove(jobId); + return answer; + default: + break; // TODO } return new UploadAnswer(jobId, getUploadPct(jobId), getUploadError(jobId), getUploadStatus2(jobId), getUploadLocalPath(jobId), getInstallPath(jobId), - getUploadTemplateSize(jobId)); + getUploadTemplateSize(jobId)); } @Override @@ -331,14 +248,11 @@ public class UploadManagerImpl extends ManagerBase implements UploadManager { String user = null; String password = null; String jobId = - uploadPublicTemplate(cmd.getId(), cmd.getUrl(), cmd.getName(), cmd.getFormat(), cmd.getAccountId(), cmd.getDescription(), cmd.getChecksum(), - cmd.getInstallPath(), user, password, cmd.getTemplateSizeInBytes()); + uploadPublicTemplate(cmd.getId(), cmd.getUrl(), cmd.getName(), cmd.getFormat(), cmd.getAccountId(), cmd.getDescription(), cmd.getChecksum(), + cmd.getInstallPath(), user, password, cmd.getTemplateSizeInBytes()); sleep(); - if (jobId == null) { - return new UploadAnswer(null, 0, "Internal Error", com.cloud.storage.UploadVO.Status.UPLOAD_ERROR, "", "", 0); - } return new UploadAnswer(jobId, getUploadPct(jobId), getUploadError(jobId), getUploadStatus2(jobId), getUploadLocalPath(jobId), getInstallPath(jobId), - getUploadTemplateSize(jobId)); + getUploadTemplateSize(jobId)); } @Override @@ -435,13 +349,10 @@ public class UploadManagerImpl extends ManagerBase implements UploadManager { } private long getUploadTemplateSize(String jobId) { - UploadJob uj = jobs.get(jobId); - if (uj != null) { - return uj.getTemplatesize(); - } return 0; } + @SuppressWarnings("unchecked") @Override public boolean configure(String name, Map params) throws ConfigurationException { @@ -466,11 +377,7 @@ public class UploadManagerImpl extends ManagerBase implements UploadManager { throw new ConfigurationException("Unable to instantiate " + value); } } - String useSsl = (String)params.get("sslcopy"); - if (useSsl != null) { - _sslCopy = Boolean.parseBoolean(useSsl); - } String inSystemVM = (String)params.get("secondary.storage.vm"); if (inSystemVM != null && "true".equalsIgnoreCase(inSystemVM)) { s_logger.info("UploadManager: starting additional services since we are inside system vm"); @@ -478,9 +385,6 @@ public class UploadManagerImpl extends ManagerBase implements UploadManager { //blockOutgoingOnPrivate(); } - value = (String)params.get("install.timeout.pergig"); - this.installTimeoutPerGig = NumbersUtil.parseInt(value, 15 * 60) * 1000; - value = (String)params.get("install.numthreads"); final int numInstallThreads = NumbersUtil.parseInt(value, 10); @@ -544,43 +448,43 @@ public class UploadManagerImpl extends ManagerBase implements UploadManager { s_logger.warn("UploadedBytes=" + tu.getUploadedBytes() + ", error=" + tu.getUploadError() + ", pct=" + tu.getUploadPercent()); switch (status) { - case ABORTED: - case NOT_STARTED: - case UNRECOVERABLE_ERROR: - // Delete the entity only if its a volume. TO DO - find a better way of finding it a volume. - if (uj.getTemplateUploader().getUploadLocalPath().indexOf("volume") > -1) { - uj.cleanup(); - } - break; - case UNKNOWN: - return; - case IN_PROGRESS: - s_logger.info("Resuming jobId: " + jobId + ", status=" + status); - tu.setResume(true); - threadPool.execute(tu); - break; - case RECOVERABLE_ERROR: - threadPool.execute(tu); - break; - case UPLOAD_FINISHED: - tu.setUploadError("Upload success, starting install "); - String result = postUpload(jobId); - if (result != null) { - s_logger.error("Failed post upload script: " + result); - tu.setStatus(Status.UNRECOVERABLE_ERROR); - tu.setUploadError("Failed post upload script: " + result); - } else { - s_logger.warn("Upload completed successfully at " + new SimpleDateFormat().format(new Date())); - tu.setStatus(Status.POST_UPLOAD_FINISHED); - tu.setUploadError("Upload completed successfully at " + new SimpleDateFormat().format(new Date())); - } - // Delete the entity only if its a volume. TO DO - find a better way of finding it a volume. - if (uj.getTemplateUploader().getUploadLocalPath().indexOf("volume") > -1) { - uj.cleanup(); - } - break; - default: - break; + case ABORTED: + case NOT_STARTED: + case UNRECOVERABLE_ERROR: + // Delete the entity only if its a volume. TO DO - find a better way of finding it a volume. + if (uj.getTemplateUploader().getUploadLocalPath().indexOf("volume") > -1) { + uj.cleanup(); + } + break; + case UNKNOWN: + return; + case IN_PROGRESS: + s_logger.info("Resuming jobId: " + jobId + ", status=" + status); + tu.setResume(true); + threadPool.execute(tu); + break; + case RECOVERABLE_ERROR: + threadPool.execute(tu); + break; + case UPLOAD_FINISHED: + tu.setUploadError("Upload success, starting install "); + String result = postUpload(jobId); + if (result != null) { + s_logger.error("Failed post upload script: " + result); + tu.setStatus(Status.UNRECOVERABLE_ERROR); + tu.setUploadError("Failed post upload script: " + result); + } else { + s_logger.warn("Upload completed successfully at " + new SimpleDateFormat().format(new Date())); + tu.setStatus(Status.POST_UPLOAD_FINISHED); + tu.setUploadError("Upload completed successfully at " + new SimpleDateFormat().format(new Date())); + } + // Delete the entity only if its a volume. TO DO - find a better way of finding it a volume. + if (uj.getTemplateUploader().getUploadLocalPath().indexOf("volume") > -1) { + uj.cleanup(); + } + break; + default: + break; } }