etan39
/
cloudstack
mirror of https://github.com/apache/cloudstack.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

CLOUDSTACK-10046: checksum validation for templates and isos when download is complete

This commit is contained in:
Daan Hoogland 2017-08-21 10:53:56 +02:00 committed by Boris
parent c4f76a199b
commit 7a9c0be1fd
16 changed files with 761 additions and 177 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
core/src/com/cloud/agent/api/ComputeChecksumCommand.java
View File

@ -25,6 +25,7 @@ import com.cloud.agent.api.to.DataStoreTO;
public class ComputeChecksumCommand extends SsCommand {
private DataStoreTO store;
private String templatePath;
private String algorithm = "MD5";
public ComputeChecksumCommand() {
super();
@ -35,6 +36,11 @@ public class ComputeChecksumCommand extends SsCommand {
this.setStore(store);
}
public ComputeChecksumCommand(DataStoreTO store, String templatePath, String algorithm) {
this(store,templatePath);
this.algorithm = algorithm;
}
public String getTemplatePath() {
return templatePath;
}
@ -43,8 +49,12 @@ public class ComputeChecksumCommand extends SsCommand {
return store;
}
public void setStore(DataStoreTO store) {
this.store = store;
public String getAlgorithm() {
return algorithm;
}
void setStore(DataStoreTO store) {
this.store = store;
}
}

2
engine/components-api/src/com/cloud/template/TemplateManager.java
View File

@ -108,7 +108,7 @@ public interface TemplateManager {
DataStore getImageStore(String storeUuid, Long zoneId);
String getChecksum(DataStore store, String templatePath);
String getChecksum(DataStore store, String templatePath, String algorithm);
List<DataStore> getImageStoreByTemplate(long templateId, Long zoneId);

15
scripts/installer/createtmplt.sh
View File

@ -38,8 +38,19 @@ fi
}
verify_cksum() {
echo "$1 $2" | md5sum -c --status
#printf "$1\t$2" | md5sum -c --status
digestalgo=""
# NOTE this will only work with 0-padded checksums
case ${#1} in
32) digestalgo="md5sum" ;;
40) digestalgo="sha1sum" ;;
56) digestalgo="sha224sum" ;;
64) digestalgo="sha256sum" ;;
96) digestalgo="sha384sum" ;;
128) digestalgo="sha512sum" ;;
*) echo "Please provide valid cheksum" ; exit 3 ;;
esac
echo "$1 $2" | $digestalgo -c --status
#printf "$1\t$2" | $digestalgo -c --status
if [ $? -gt 0 ]
then
printf "Checksum failed, not proceeding with install\n"

17
scripts/installer/createvolume.sh
View File

@ -39,9 +39,20 @@ fi
}
verify_cksum() {
echo "$1 $2" | md5sum -c --status
#printf "$1\t$2" | md5sum -c --status
if [ $? -gt 0 ]
digestalgo=""
# NOTE this will only work with 0-padded checksums
case ${#1} in
32) digestalgo="md5sum" ;;
40) digestalgo="sha1sum" ;;
56) digestalgo="sha224sum" ;;
64) digestalgo="sha256sum" ;;
96) digestalgo="sha384sum" ;;
128) digestalgo="sha512sum" ;;
*) echo "Please provide valid cheksum" ; exit 3 ;;
esac
echo "$1 $2" | $digestalgo -c --status
#printf "$1\t$2" | $digestalgo -c --status
if [ $? -gt 0 ]
then
printf "Checksum failed, not proceeding with install\n"
exit 3

14
scripts/storage/qcow2/createtmplt.sh
View File

@ -21,7 +21,7 @@
# createtmplt.sh -- install a template
usage() {
printf "Usage: %s: -t <template-fs> -n <templatename> -f <root disk file> -s <size in Gigabytes> -c <md5 cksum> -d <descr> -h [-u]\n" $(basename $0) >&2
printf "Usage: %s: -t <template-fs> -n <templatename> -f <root disk file> -s <size in Gigabytes> -c <snapshot name> -d <descr> -h [-u]\n" $(basename $0) >&2
}
@ -37,17 +37,6 @@ then
fi
fi
verify_cksum() {
echo "$1 $2" | md5sum -c --status
#printf "$1\t$2" | md5sum -c --status
if [ $? -gt 0 ]
then
printf "Checksum failed, not proceeding with install\n"
exit 3
fi
}
untar() {
local ft=$(file $1| awk -F" " '{print $2}')
local basedir=$(dirname $1)
@ -156,7 +145,6 @@ do
tmpltimg="$OPTARG"
;;
s) sflag=1
sflag=1
;;
c) cflag=1
snapshotName="$OPTARG"

14
scripts/storage/qcow2/createvolume.sh
View File

@ -22,7 +22,7 @@
# createvol.sh -- install a volume
usage() {
printf "Usage: %s: -t <volume-fs> -n <volumename> -f <root disk file> -s <size in Gigabytes> -c <md5 cksum> -d <descr> -h [-u]\n" $(basename $0) >&2
printf "Usage: %s: -t <volume-fs> -n <volumename> -f <root disk file> -s <size in Gigabytes> -c <snapshot name> -d <descr> -h [-u]\n" $(basename $0) >&2
}
@ -38,17 +38,6 @@ then
fi
fi
verify_cksum() {
echo "$1 $2" | md5sum -c --status
#printf "$1\t$2" | md5sum -c --status
if [ $? -gt 0 ]
then
printf "Checksum failed, not proceeding with install\n"
exit 3
fi
}
untar() {
local ft=$(file $1| awk -F" " '{print $2}')
local basedir=$(dirname $1)
@ -157,7 +146,6 @@ do
volimg="$OPTARG"
;;
s) sflag=1
sflag=1
;;
c) cflag=1
snapshotName="$OPTARG"

24
scripts/storage/secondary/createtmplt.sh
View File

@ -22,7 +22,7 @@
# createtmplt.sh -- install a template
usage() {
printf "Usage: %s: -t <template-fs> -n <templatename> -f <root disk file> -c <md5 cksum> -d <descr> -h [-u] [-v]\n" $(basename $0) >&2
printf "Usage: %s: -t <template-fs> -n <templatename> -f <root disk file> -d <descr> -h [-u] [-v]\n" $(basename $0) >&2
}
@ -40,16 +40,6 @@ rollback_if_needed() {
fi
}
verify_cksum() {
echo "$1 $2" | md5sum -c --status
#printf "$1\t$2" | md5sum -c --status
if [ $? -gt 0 ]
then
printf "Checksum failed, not proceeding with install\n"
exit 3
fi
}
untar() {
local ft=$(file $1| awk -F" " '{print $2}')
case $ft in
@ -129,9 +119,8 @@ hflag=
hvm=false
cleanup=false
dflag=
cflag=
while getopts 'vuht:n:f:s:c:d:S:' OPTION
while getopts 'vuht:n:f:s:d:S:' OPTION
do
case $OPTION in
t) tflag=1
@ -145,9 +134,6 @@ do
;;
s) sflag=1
;;
c) cflag=1
cksum="$OPTARG"
;;
d) dflag=1
descr="$OPTARG"
;;
@ -191,10 +177,6 @@ then
exit 3
fi
if [ -n "$cksum" ]
then
verify_cksum $cksum $tmpltimg
fi
[ -n "$verbose" ] && is_compressed $tmpltimg
tmpltimg2=$(uncompress $tmpltimg)
rollback_if_needed $tmpltfs $? "failed to uncompress $tmpltimg\n"
@ -227,6 +209,8 @@ echo -n "" > /$tmpltfs/template.properties
today=$(date '+%m_%d_%Y')
echo "filename=$tmpltname" > /$tmpltfs/template.properties
echo "description=$descr" >> /$tmpltfs/template.properties
# we need to rethink this property as it might get changed after download due to decompression
# option is to recalcutate it here
echo "checksum=$cksum" >> /$tmpltfs/template.properties
echo "hvm=$hvm" >> /$tmpltfs/template.properties
echo "size=$imgsize" >> /$tmpltfs/template.properties

15
scripts/storage/secondary/createvolume.sh
View File

@ -41,8 +41,19 @@ fi
}
verify_cksum() {
echo "$1 $2" | md5sum -c --status
#printf "$1\t$2" | md5sum -c --status
digestalgo=""
# NOTE this will only work with 0-padded checksums
case ${#1} in
32) digestalgo="md5sum" ;;
40) digestalgo="sha1sum" ;;
56) digestalgo="sha224sum" ;;
64) digestalgo="sha256sum" ;;
96) digestalgo="sha384sum" ;;
128) digestalgo="sha512sum" ;;
*) echo "Please provide valid cheksum" ; exit 3 ;;
esac
echo "$1 $2" | $digestalgo -c --status
#printf "$1\t$2" | $digestalgo -c --status
if [ $? -gt 0 ]
then
printf "Checksum failed, not proceeding with install\n"

4
server/src/com/cloud/template/TemplateManagerImpl.java
View File

@ -570,9 +570,9 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
}
@Override
public String getChecksum(DataStore store, String templatePath) {
public String getChecksum(DataStore store, String templatePath, String algorithm) {
EndPoint ep = _epSelector.select(store);
ComputeChecksumCommand cmd = new ComputeChecksumCommand(store.getTO(), templatePath);
ComputeChecksumCommand cmd = new ComputeChecksumCommand(store.getTO(), templatePath, algorithm);
Answer answer = null;
if (ep == null) {
String errMsg = "No remote endpoint to send command, check if host or ssvm is down?";

124
services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java
View File

@ -16,13 +16,6 @@
// under the License.
package org.apache.cloudstack.storage.resource;
import static com.cloud.utils.S3Utils.mputFile;
import static com.cloud.utils.S3Utils.putFile;
import static com.cloud.utils.StringUtils.join;
import static java.lang.String.format;
import static java.util.Arrays.asList;
import static org.apache.commons.lang.StringUtils.substringAfterLast;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
@ -32,11 +25,9 @@ import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.math.BigInteger;
import java.net.InetAddress;
import java.net.URI;
import java.net.UnknownHostException;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.HashMap;
@ -46,34 +37,7 @@ import java.util.UUID;
import javax.naming.ConfigurationException;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.NameValuePair;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.utils.URLEncodedUtils;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.log4j.Logger;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import org.apache.cloudstack.framework.security.keystore.KeystoreManager;
import org.apache.cloudstack.storage.command.CopyCmdAnswer;
import org.apache.cloudstack.storage.command.CopyCommand;
import org.apache.cloudstack.storage.command.DeleteCommand;
import org.apache.cloudstack.storage.command.DownloadCommand;
import org.apache.cloudstack.storage.command.DownloadProgressCommand;
import org.apache.cloudstack.storage.template.DownloadManager;
import org.apache.cloudstack.storage.template.DownloadManagerImpl;
import org.apache.cloudstack.storage.template.DownloadManagerImpl.ZfsPathParser;
import org.apache.cloudstack.storage.template.UploadManager;
import org.apache.cloudstack.storage.template.UploadManagerImpl;
import org.apache.cloudstack.storage.to.SnapshotObjectTO;
import org.apache.cloudstack.storage.to.TemplateObjectTO;
import org.apache.cloudstack.storage.to.VolumeObjectTO;
import com.cloud.agent.api.Answer;
import com.cloud.agent.api.CheckHealthAnswer;
import com.cloud.agent.api.CheckHealthCommand;
@ -135,6 +99,38 @@ import com.cloud.utils.net.NetUtils;
import com.cloud.utils.script.OutputInterpreter;
import com.cloud.utils.script.Script;
import com.cloud.vm.SecondaryStorageVm;
import org.apache.cloudstack.framework.security.keystore.KeystoreManager;
import org.apache.cloudstack.storage.command.CopyCmdAnswer;
import org.apache.cloudstack.storage.command.CopyCommand;
import org.apache.cloudstack.storage.command.DeleteCommand;
import org.apache.cloudstack.storage.command.DownloadCommand;
import org.apache.cloudstack.storage.command.DownloadProgressCommand;
import org.apache.cloudstack.storage.template.DownloadManager;
import org.apache.cloudstack.storage.template.DownloadManagerImpl;
import org.apache.cloudstack.storage.template.DownloadManagerImpl.ZfsPathParser;
import org.apache.cloudstack.storage.template.UploadManager;
import org.apache.cloudstack.storage.template.UploadManagerImpl;
import org.apache.cloudstack.storage.to.SnapshotObjectTO;
import org.apache.cloudstack.storage.to.TemplateObjectTO;
import org.apache.cloudstack.storage.to.VolumeObjectTO;
import org.apache.cloudstack.utils.security.DigestHelper;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.NameValuePair;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.utils.URLEncodedUtils;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.log4j.Logger;
import static com.cloud.utils.S3Utils.mputFile;
import static com.cloud.utils.S3Utils.putFile;
import static com.cloud.utils.StringUtils.join;
import static java.lang.String.format;
import static java.util.Arrays.asList;
import static org.apache.commons.lang.StringUtils.substringAfterLast;
public class NfsSecondaryStorageResource extends ServerResourceBase implements SecondaryStorageResource {
@ -654,7 +650,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
}
private String determineS3TemplateNameFromKey(String key) {
return StringUtils.substringAfterLast(StringUtils.substringBeforeLast(key, S3Utils.SEPARATOR), S3Utils.SEPARATOR);
return substringAfterLast(StringUtils.substringBeforeLast(key, S3Utils.SEPARATOR), S3Utils.SEPARATOR);
}
@SuppressWarnings("unchecked")
@ -663,7 +659,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
}
protected Long determineS3VolumeIdFromKey(String key) {
return Long.parseLong(StringUtils.substringAfterLast(StringUtils.substringBeforeLast(key, S3Utils.SEPARATOR), S3Utils.SEPARATOR));
return Long.parseLong(substringAfterLast(StringUtils.substringBeforeLast(key, S3Utils.SEPARATOR), S3Utils.SEPARATOR));
}
private String determineStorageTemplatePath(final String storagePath, String dataPath) {
@ -1096,16 +1092,16 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
final String bucket = s3.getBucketName();
try {
S3Utils.deleteDirectory(s3, bucket, path);
return new Answer(cmd, true, String.format("Deleted snapshot %1%s from bucket %2$s.", path, bucket));
return new Answer(cmd, true, format("Deleted snapshot %1%s from bucket %2$s.", path, bucket));
} catch (Exception e) {
final String errorMessage =
String.format("Failed to delete snapshot %1$s from bucket %2$s due to the following error: %3$s", path, bucket, e.getMessage());
format("Failed to delete snapshot %1$s from bucket %2$s due to the following error: %3$s", path, bucket, e.getMessage());
s_logger.error(errorMessage, e);
return new Answer(cmd, false, errorMessage);
}
} else if (dstore instanceof SwiftTO) {
String path = cmd.getDirectory();
String volumeId = StringUtils.substringAfterLast(path, "/"); // assuming
String volumeId = substringAfterLast(path, "/"); // assuming
// that
// the
// filename
@ -1146,46 +1142,24 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
parent += File.separator;
}
String absoluteTemplatePath = parent + relativeTemplatePath;
MessageDigest digest;
String checksum = null;
String algorithm = cmd.getAlgorithm();
File f = new File(absoluteTemplatePath);
InputStream is = null;
byte[] buffer = new byte[8192];
int read = 0;
if (s_logger.isDebugEnabled()) {
s_logger.debug("parent path " + parent + " relative template path " + relativeTemplatePath);
}
String checksum = null;
try {
digest = MessageDigest.getInstance("MD5");
is = new FileInputStream(f);
while ((read = is.read(buffer)) > 0) {
digest.update(buffer, 0, read);
}
byte[] md5sum = digest.digest();
BigInteger bigInt = new BigInteger(1, md5sum);
checksum = bigInt.toString(16);
try (InputStream is = new FileInputStream(f);){
checksum = DigestHelper.digest(algorithm, is).toString();
if (s_logger.isDebugEnabled()) {
s_logger.debug("Successfully calculated checksum for file " + absoluteTemplatePath + " - " + checksum);
}
} catch (IOException e) {
String logMsg = "Unable to process file for MD5 - " + absoluteTemplatePath;
String logMsg = "Unable to process file for " + algorithm + " - " + absoluteTemplatePath;
s_logger.error(logMsg);
return new Answer(cmd, false, checksum);
} catch (NoSuchAlgorithmException e) {
return new Answer(cmd, false, checksum);
} finally {
try {
if (is != null) {
is.close();
}
} catch (IOException e) {
if (s_logger.isDebugEnabled()) {
s_logger.debug("Could not close the file " + absoluteTemplatePath);
}
return new Answer(cmd, false, checksum);
}
}
return new Answer(cmd, true, checksum);
@ -1330,10 +1304,10 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
final String bucket = s3.getBucketName();
try {
S3Utils.deleteObject(s3, bucket, path);
return new Answer(cmd, true, String.format("Deleted snapshot %1%s from bucket %2$s.", path, bucket));
return new Answer(cmd, true, format("Deleted snapshot %1%s from bucket %2$s.", path, bucket));
} catch (Exception e) {
final String errorMessage =
String.format("Failed to delete snapshot %1$s from bucket %2$s due to the following error: %3$s", path, bucket, e.getMessage());
format("Failed to delete snapshot %1$s from bucket %2$s due to the following error: %3$s", path, bucket, e.getMessage());
s_logger.error(errorMessage, e);
return new Answer(cmd, false, errorMessage);
}
@ -1689,10 +1663,10 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
final String bucket = s3.getBucketName();
try {
S3Utils.deleteDirectory(s3, bucket, path);
return new Answer(cmd, true, String.format("Deleted template %1$s from bucket %2$s.", path, bucket));
return new Answer(cmd, true, format("Deleted template %1$s from bucket %2$s.", path, bucket));
} catch (Exception e) {
final String errorMessage =
String.format("Failed to delete template %1$s from bucket %2$s due to the following error: %3$s", path, bucket, e.getMessage());
format("Failed to delete template %1$s from bucket %2$s due to the following error: %3$s", path, bucket, e.getMessage());
s_logger.error(errorMessage, e);
return new Answer(cmd, false, errorMessage);
}
@ -1793,16 +1767,16 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
final String bucket = s3.getBucketName();
try {
S3Utils.deleteDirectory(s3, bucket, path);
return new Answer(cmd, true, String.format("Deleted volume %1%s from bucket %2$s.", path, bucket));
return new Answer(cmd, true, format("Deleted volume %1%s from bucket %2$s.", path, bucket));
} catch (Exception e) {
final String errorMessage = String.format("Failed to delete volume %1$s from bucket %2$s due to the following error: %3$s", path, bucket, e.getMessage());
final String errorMessage = format("Failed to delete volume %1$s from bucket %2$s due to the following error: %3$s", path, bucket, e.getMessage());
s_logger.error(errorMessage, e);
return new Answer(cmd, false, errorMessage);
}
} else if (dstore instanceof SwiftTO) {
Long volumeId = obj.getId();
String path = obj.getPath();
String filename = StringUtils.substringAfterLast(path, "/"); // assuming
String filename = substringAfterLast(path, "/"); // assuming
// that
// the
// filename

60
services/secondary-storage/server/src/org/apache/cloudstack/storage/template/DownloadManagerImpl.java
View File

@ -21,11 +21,9 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.math.BigInteger;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.charset.Charset;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
@ -42,11 +40,14 @@ import java.util.concurrent.Executors;
import javax.ejb.Local;
import javax.naming.ConfigurationException;
import com.cloud.utils.StringUtils;
import org.apache.cloudstack.storage.command.DownloadCommand;
import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType;
import org.apache.cloudstack.storage.command.DownloadProgressCommand;
import org.apache.cloudstack.storage.command.DownloadProgressCommand.RequestType;
import org.apache.cloudstack.storage.resource.SecondaryStorageResource;
import org.apache.cloudstack.utils.security.ChecksumValue;
import org.apache.cloudstack.utils.security.DigestHelper;
import org.apache.log4j.Logger;
import com.cloud.agent.api.storage.DownloadAnswer;
@ -309,33 +310,13 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
}
}
private String computeCheckSum(File f) {
byte[] buffer = new byte[8192];
int read = 0;
MessageDigest digest;
String checksum = null;
InputStream is = null;
try {
digest = MessageDigest.getInstance("MD5");
is = new FileInputStream(f);
while ((read = is.read(buffer)) > 0) {
digest.update(buffer, 0, read);
}
byte[] md5sum = digest.digest();
BigInteger bigInt = new BigInteger(1, md5sum);
checksum = String.format("%032x", bigInt);
return checksum;
private ChecksumValue computeCheckSum(String algorithm, File f) {
try (InputStream is = new FileInputStream(f);) {
return DigestHelper.digest(algorithm, is);
} catch (IOException e) {
return null;
} catch (NoSuchAlgorithmException e) {
return null;
} finally {
try {
if (is != null)
is.close();
} catch (IOException e) {
return null;
}
}
}
@ -392,11 +373,17 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
ResourceType resourceType = dnld.getResourceType();
File originalTemplate = new File(td.getDownloadLocalPath());
String checkSum = computeCheckSum(originalTemplate);
if (checkSum == null) {
ChecksumValue oldValue = new ChecksumValue(dnld.getChecksum());
ChecksumValue newValue = computeCheckSum(oldValue.getAlgorithm(), originalTemplate);
if(StringUtils.isNotBlank(dnld.getChecksum()) && ! oldValue.equals(newValue)) {
return "checksum \"" + newValue +"\" didn't match the given value, \"" + oldValue + "\"";
}
String checksum = newValue.getChecksum();
if (checksum == null) {
s_logger.warn("Something wrong happened when try to calculate the checksum of downloaded template!");
}
dnld.setCheckSum(checkSum);
dnld.setCheckSum(checksum);
int imgSizeGigs = (int)Math.ceil(_storage.getSize(td.getDownloadLocalPath()) * 1.0d / (1024 * 1024 * 1024));
imgSizeGigs++; // add one just in case
@ -429,11 +416,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
scr.add("-n", templateFilename);
scr.add("-t", resourcePath);
scr.add("-f", td.getDownloadLocalPath()); // this is the temporary
// template file downloaded
if (dnld.getChecksum() != null && dnld.getChecksum().length() > 1) {
scr.add("-c", dnld.getChecksum());
}
scr.add("-f", td.getDownloadLocalPath()); // this is the temporary template file downloaded
scr.add("-u"); // cleanup
String result;
result = scr.execute();
@ -854,17 +837,6 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
result.put(tInfo.getTemplateName(), tInfo);
s_logger.debug("Added template name: " + tInfo.getTemplateName() + ", path: " + tmplt);
}
/*
for (String tmplt : isoTmplts) {
String tmp[];
tmp = tmplt.split("/");
String tmpltName = tmp[tmp.length - 2];
tmplt = tmplt.substring(tmplt.lastIndexOf("iso/"));
TemplateInfo tInfo = new TemplateInfo(tmpltName, tmplt, false);
s_logger.debug("Added iso template name: " + tmpltName + ", path: " + tmplt);
result.put(tmpltName, tInfo);
}
*/
return result;
}

162
test/integration/smoke/test_iso.py
View File

@ -546,3 +546,165 @@ class TestISO(cloudstackTestCase):
return
class TestCreateISOWithChecksum(cloudstackTestCase):
def setUp(self):
self.testClient = super(TestCreateISOWithChecksum, self).getClsTestClient()
self.apiclient = self.testClient.getApiClient()
self.cleanup = []
self.unsupportedHypervisor = False
self.hypervisor = self.testClient.getHypervisorInfo()
if self.hypervisor.lower() in ['lxc']:
# Template creation from root volume is not supported in LXC
self.unsupportedHypervisor = True
return
# Get Zone, Domain and templates
self.zone = get_zone(self.apiclient, self.testClient.getZoneForTests())
# Setup default create iso attributes
self.iso = registerIso.registerIsoCmd()
self.iso.checksum = "{SHA-1}" + "e16f703b5d6cb6dd2c448d956be63fcbee7d79ea"
self.iso.zoneid = self.zone.id
self.iso.name = 'test-tynyCore-iso'
self.iso.displaytext = 'test-tynyCore-iso'
self.iso.url = "http://dl.openvm.eu/cloudstack/iso/TinyCore-8.0.iso"
self.iso.ostypeid = self.getOsType("Other Linux (64-bit)")
self.md5 = "f7fee34a73a7f8e3adb30778c7c32c51"
self.sha256 = "069a22f7cc15b34cd39f6dd61ef0cf99ff47a1a92942772c30f50988746517f7"
if self.unsupportedHypervisor:
self.skipTest("Skipping test because unsupported hypervisor\
%s" % self.hypervisor)
return
def tearDown(self):
try:
# Clean up the created templates
for temp in self.cleanup:
cmd = deleteIso.deleteIsoCmd()
cmd.id = temp.id
cmd.zoneid = self.zone.id
self.apiclient.deleteIso(cmd)
except Exception as e:
raise Exception("Warning: Exception during cleanup : %s" % e)
return
@attr(tags=["advanced", "smoke"], required_hardware="true")
def test_01_create_iso_with_checksum_sha1(self):
iso = self.registerIso(self.iso)
self.download(self.apiclient, iso.id)
@attr(tags=["advanced", "smoke"], required_hardware="true")
def test_02_create_iso_with_checksum_sha256(self):
self.iso.checksum = "{SHA-256}" + self.sha256
iso = self.registerIso(self.iso)
self.download(self.apiclient, iso.id)
@attr(tags=["advanced", "smoke"], required_hardware="true")
def test_03_create_iso_with_checksum_md5(self):
self.iso.checksum = "{md5}" + self.md5
iso = self.registerIso(self.iso)
self.download(self.apiclient, iso.id)
@attr(tags=["advanced", "smoke"], required_hardware="true")
def test_01_1_create_iso_with_checksum_sha1_negative(self):
self.iso.checksum = "{sha-1}" + "someInvalidValue"
iso = self.registerIso(self.iso)
try:
self.download(self.apiclient, iso.id)
except Exception as e:
print "Negative Test Passed - Exception Occurred Under iso download " \
"%s" % GetDetailExceptionInfo(e)
else:
self.fail("Negative Test Failed - Exception DID NOT Occurred Under iso download ")
@attr(tags=["advanced", "smoke"], required_hardware="true")
def test_02_1_create_iso_with_checksum_sha256_negative(self):
self.iso.checksum = "{SHA-256}" + "someInvalidValue"
iso = self.registerIso(self.iso)
try:
self.download(self.apiclient, iso.id)
except Exception as e:
print "Negative Test Passed - Exception Occurred Under iso download " \
"%s" % GetDetailExceptionInfo(e)
else:
self.fail("Negative Test Failed - Exception DID NOT Occurred Under iso download ")
@attr(tags=["advanced", "smoke"], required_hardware="true")
def test_03_1_create_iso_with_checksum_md5_negative(self):
self.iso.checksum = "{md5}" + "someInvalidValue"
iso = self.registerIso(self.iso)
try:
self.download(self.apiclient, iso.id)
except Exception as e:
print "Negative Test Passed - Exception Occurred Under iso download " \
"%s" % GetDetailExceptionInfo(e)
else:
self.fail("Negative Test Failed - Exception DID NOT Occurred Under iso download ")
@attr(tags=["advanced", "smoke"], required_hardware="true")
def test_04_create_iso_with_no_checksum(self):
self.iso.checksum = None
iso = self.registerIso(self.iso)
self.download(self.apiclient, iso.id)
def registerIso(self, cmd):
iso = self.apiclient.registerIso(cmd)[0]
self.cleanup.append(iso)
return iso
def getOsType(self, param):
cmd = listOsTypes.listOsTypesCmd()
cmd.description = param
return self.apiclient.listOsTypes(cmd)[0].id
def download(self, apiclient, iso_id, retries=12, interval=5):
"""Check if template download will finish in 1 minute"""
while retries > -1:
time.sleep(interval)
iso_response = Iso.list(
apiclient,
id=iso_id
)
if isinstance(iso_response, list):
iso = iso_response[0]
if not hasattr(iso, 'status') or not iso or not iso.status:
retries = retries - 1
continue
# If iso is ready,
# iso.status = Download Complete
# Downloading - x% Downloaded
# if Failed
# Error - Any other string
if 'Failed' in iso.status:
raise Exception(
"Failed to download iso: status - %s" %
iso.status)
elif iso.status == 'Successfully Installed' and iso.isready:
return
elif 'Downloaded' in iso.status:
retries = retries - 1
continue
elif 'Installing' not in iso.status:
if retries >= 0:
retries = retries - 1
continue
raise Exception(
"Error in downloading iso: status - %s" %
iso.status)
else:
retries = retries - 1
raise Exception("Template download failed exception.")

201
test/integration/smoke/test_templates.py
View File

@ -18,6 +18,8 @@
"""
#Import Local Modules
from marvin.codes import FAILED
from marvin.cloudstackException import *
from marvin.cloudstackAPI import *
from marvin.cloudstackTestCase import cloudstackTestCase, unittest
from marvin.lib.utils import random_gen, cleanup_resources
from marvin.lib.base import (Account,
@ -37,6 +39,205 @@ import time
_multiprocess_shared_ = True
class TestCreateTemplateWithChecksum(cloudstackTestCase):
def setUp(self):
self.testClient = super(TestCreateTemplateWithChecksum, self).getClsTestClient()
self.apiclient = self.testClient.getApiClient()
self.dbclient = self.testClient.getDbConnection()
self.cleanup = []
self.services = self.testClient.getParsedTestDataConfig()
self.unsupportedHypervisor = False
self.hypervisor = self.testClient.getHypervisorInfo()
if self.hypervisor.lower() in ['lxc']:
# Template creation from root volume is not supported in LXC
self.unsupportedHypervisor = True
return
# Get Zone, Domain and templates
self.domain = get_domain(self.apiclient)
self.zone = get_zone(self.apiclient, self.testClient.getZoneForTests())
if "kvm" in self.hypervisor.lower():
self.test_template = registerTemplate.registerTemplateCmd()
self.test_template = registerTemplate.registerTemplateCmd()
self.test_template.checksum = "{SHA-1}" + "bf580a13f791d86acf3449a7b457a91a14389264"
self.test_template.hypervisor = self.hypervisor
self.test_template.zoneid = self.zone.id
self.test_template.name = 'test sha-2333'
self.test_template.displaytext = 'test sha-1'
self.test_template.url = "http://dl.openvm.eu/cloudstack/macchinina/x86_64/macchinina-kvm.qcow2.bz2"
self.test_template.format = "QCOW2"
self.test_template.ostypeid = self.getOsType("Other Linux (64-bit)")
self.md5 = "ada77653dcf1e59495a9e1ac670ad95f"
self.sha256 = "0efc03633f2b8f5db08acbcc5dc1be9028572dfd8f1c6c8ea663f0ef94b458c5"
if "vmware" in self.hypervisor.lower():
self.test_template = registerTemplate.registerTemplateCmd()
self.test_template = registerTemplate.registerTemplateCmd()
self.test_template.checksum = "{SHA-1}" + "b25d404de8335b4348ff01e49a95b403c90df466"
self.test_template.hypervisor = self.hypervisor
self.test_template.zoneid = self.zone.id
self.test_template.name = 'test sha-2333'
self.test_template.displaytext = 'test sha-1'
self.test_template.url = "http://dl.openvm.eu/cloudstack/macchinina/x86_64/macchinina-vmware.ova"
self.test_template.format = "OVA"
self.test_template.ostypeid = self.getOsType("Other Linux (64-bit)")
self.md5 = "d6d97389b129c7d898710195510bf4fb"
self.sha256 = "f57b59f118ab59284a70d6c63229d1de8f2d69bffc5a82b773d6c47e769c12d9"
if "xen" in self.hypervisor.lower():
self.test_template = registerTemplate.registerTemplateCmd()
self.test_template = registerTemplate.registerTemplateCmd()
self.test_template.checksum = "{SHA-1}" + "427fad501d0d8a1d63b8600a9a469fbf91191314"
self.test_template.hypervisor = self.hypervisor
self.test_template.zoneid = self.zone.id
self.test_template.name = 'test sha-2333'
self.test_template.displaytext = 'test sha-1'
self.test_template.url = "http://dl.openvm.eu/cloudstack/macchinina/x86_64/macchinina-xen.vhd.bz2"
self.test_template.format = "VHD"
self.test_template.ostypeid = self.getOsType("Other Linux (64-bit)")
self.md5 = "54ebc933e6e07ae58c0dc97dfd37c824"
self.sha256 = "bddd9876021d33df9792b71ae4b776598680ac68ecf55e9d9af33c80904cc1f3"
if self.unsupportedHypervisor:
self.skipTest("Skipping test because unsupported hypervisor\
%s" % self.hypervisor)
return
def tearDown(self):
try:
# Clean up the created templates
for temp in self.cleanup:
cmd = deleteTemplate.deleteTemplateCmd()
cmd.id = temp.id
cmd.zoneid = self.zone.id
self.apiclient.deleteTemplate(cmd)
except Exception as e:
raise Exception("Warning: Exception during cleanup : %s" % e)
return
@attr(tags=["advanced", "smoke"], required_hardware="true")
def test_02_create_template_with_checksum_sha1(self):
template = self.registerTemplate(self.test_template)
self.download(self.apiclient, template.id)
@attr(tags=["advanced", "smoke"], required_hardware="true")
def test_03_create_template_with_checksum_sha256(self):
self.test_template.checksum = "{SHA-256}" + self.sha256
template = self.registerTemplate(self.test_template)
self.download(self.apiclient, template.id)
@attr(tags=["advanced", "smoke"], required_hardware="true")
def test_04_create_template_with_checksum_md5(self):
self.test_template.checksum = "{md5}" + self.md5
template = self.registerTemplate(self.test_template)
self.download(self.apiclient, template.id)
@attr(tags=["advanced", "smoke"], required_hardware="true")
def test_02_1_create_template_with_checksum_sha1_negative(self):
self.test_template.checksum = "{sha-1}" + "someInvalidValue"
template = self.registerTemplate(self.test_template)
try:
self.download(self.apiclient, template.id)
except Exception as e:
print "Negative Test Passed - Exception Occurred Under template download " \
"%s" % GetDetailExceptionInfo(e)
else:
self.fail("Negative Test Failed - Exception DID NOT Occurred Under template download ")
@attr(tags=["advanced", "smoke"], required_hardware="true")
def test_03_1_create_template_with_checksum_sha256_negative(self):
self.test_template.checksum = "{SHA-256}" + "someInvalidValue"
template = self.registerTemplate(self.test_template)
try:
self.download(self.apiclient, template.id)
except Exception as e:
print "Negative Test Passed - Exception Occurred Under template download " \
"%s" % GetDetailExceptionInfo(e)
else:
self.fail("Negative Test Failed - Exception DID NOT Occurred Under template download ")
@attr(tags=["advanced", "smoke"], required_hardware="true")
def test_04_1_create_template_with_checksum_md5_negative(self):
self.test_template.checksum = "{md5}" + "someInvalidValue"
template = self.registerTemplate(self.test_template)
try:
self.download(self.apiclient, template.id)
except Exception as e:
print "Negative Test Passed - Exception Occurred Under template download " \
"%s" % GetDetailExceptionInfo(e)
else:
self.fail("Negative Test Failed - Exception DID NOT Occurred Under template download ")
@attr(tags=["advanced", "smoke"], required_hardware="true")
def test_05_create_template_with_no_checksum(self):
self.test_template.checksum = None
template = self.registerTemplate(self.test_template)
self.download(self.apiclient, template.id)
def registerTemplate(self, cmd):
temp = self.apiclient.registerTemplate(cmd)[0]
self.cleanup.append(temp)
return temp
def getOsType(self, param):
cmd = listOsTypes.listOsTypesCmd()
cmd.description = param
return self.apiclient.listOsTypes(cmd)[0].id
def download(self, apiclient, template_id, retries=12, interval=5):
"""Check if template download will finish in 1 minute"""
while retries > -1:
time.sleep(interval)
template_response = Template.list(
apiclient,
id=template_id,
zoneid=self.zone.id,
templatefilter='self'
)
if isinstance(template_response, list):
template = template_response[0]
if not hasattr(template, 'status') or not template or not template.status:
retries = retries - 1
continue
# If template is ready,
# template.status = Download Complete
# Downloading - x% Downloaded
# if Failed
# Error - Any other string
if 'Failed' in template.status:
raise Exception(
"Failed to download template: status - %s" %
template.status)
elif template.status == 'Download Complete' and template.isready:
return
elif 'Downloaded' in template.status:
retries = retries - 1
continue
elif 'Installing' not in template.status:
if retries >= 0:
retries = retries - 1
continue
raise Exception(
"Error in downloading template: status - %s" %
template.status)
else:
retries = retries - 1
raise Exception("Template download failed exception.")
class TestCreateTemplate(cloudstackTestCase):
def setUp(self):

89
utils/src/org/apache/cloudstack/utils/security/ChecksumValue.java Normal file
View File

@ -0,0 +1,89 @@
//
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//
package org.apache.cloudstack.utils.security;
import java.util.Objects;
import org.apache.commons.lang.StringUtils;
public class ChecksumValue {
String checksum;
String algorithm = "MD5";
public ChecksumValue(String algorithm, String checksum) {
this.algorithm = algorithm;
this.checksum = checksum;
}
public ChecksumValue(String digest) {
digest = StringUtils.strip(digest);
this.algorithm = algorithmFromDigest(digest);
this.checksum = stripAlgorithmFromDigest(digest);
}
@Override
public String toString() {
return '{' + algorithm + '}'+ checksum;
}
@Override
public boolean equals(Object o) {
if (this == o)
return true;
if (o == null || getClass() != o.getClass())
return false;
ChecksumValue that = (ChecksumValue)o;
return Objects.equals(getChecksum(), that.getChecksum()) && Objects.equals(getAlgorithm(), that.getAlgorithm());
}
@Override
public int hashCode() {
return Objects.hash(getChecksum(), getAlgorithm());
}
public String getChecksum() {
return checksum;
}
public String getAlgorithm() {
return algorithm;
}
private static String stripAlgorithmFromDigest(String digest) {
if(StringUtils.isNotEmpty(digest)) {
int s = digest.indexOf('{');// only assume a
int e = digest.indexOf('}');
if (s == 0 && e > s) { // we have an algorithm name of at least 1 char
return digest.substring(e+1);
}
}
// we assume digest is alright if there is no algorithm at the start
return digest;
}
private static String algorithmFromDigest(String digest) {
if(StringUtils.isNotEmpty(digest)) {
int s = digest.indexOf('{');
int e = digest.indexOf('}');
if (s == 0 && e > s+1) { // we have an algorithm name of at least 1 char
return digest.substring(s+1,e);
} // else if no algoritm
} // or if no digest at all
return "MD5";
}
}

73
utils/src/org/apache/cloudstack/utils/security/DigestHelper.java Normal file
View File

@ -0,0 +1,73 @@
//
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//
package org.apache.cloudstack.utils.security;
import java.io.IOException;
import java.io.InputStream;
import java.math.BigInteger;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
public class DigestHelper {
public static ChecksumValue digest(String algorithm, InputStream is) throws NoSuchAlgorithmException, IOException {
MessageDigest digest = MessageDigest.getInstance(algorithm);
ChecksumValue checksum = null;
byte[] buffer = new byte[8192];
int read = 0;
while ((read = is.read(buffer)) > 0) {
digest.update(buffer, 0, read);
}
byte[] md5sum = digest.digest();
// TODO make sure this is valid for all types of checksums !?!
BigInteger bigInt = new BigInteger(1, md5sum);
checksum = new ChecksumValue(digest.getAlgorithm(), getPaddedDigestString(digest,bigInt));
return checksum;
}
public static boolean check(String checksum, InputStream is) throws IOException, NoSuchAlgorithmException {
ChecksumValue toCheckAgainst = new ChecksumValue(checksum);
String algorithm = toCheckAgainst.getAlgorithm();
ChecksumValue result = digest(algorithm,is);
return result.equals(toCheckAgainst);
}
public static String getPaddedDigest(String algorithm, String inputString) throws NoSuchAlgorithmException {
MessageDigest digest = MessageDigest.getInstance(algorithm);
String checksum;
digest.reset();
BigInteger pwInt = new BigInteger(1, digest.digest(inputString.getBytes()));
return getPaddedDigestString(digest, pwInt);
}
private static String getPaddedDigestString(MessageDigest digest, BigInteger pwInt) {
String checksum;
String pwStr = pwInt.toString(16);
// we have half byte string representation, so
int padding = 2*digest.getDigestLength() - pwStr.length();
StringBuffer sb = new StringBuffer();
for (int i = 0; i < padding; i++) {
sb.append('0'); // make sure the MD5 password is 32 digits long
}
sb.append(pwStr);
checksum = sb.toString();
return checksum;
}
}

110
utils/test/org/apache/cloudstack/utils/security/DigestHelperTest.java Normal file
View File

@ -0,0 +1,110 @@
//
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//
package org.apache.cloudstack.utils.security;
import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import com.amazonaws.util.StringInputStream;
import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
public class DigestHelperTest {
private final static String INPUT_STRING = "01234567890123456789012345678901234567890123456789012345678901234567890123456789\n";
private final static String INPUT_STRING_NO2 = "01234567890123456789012345678901234567890123456789012345678901234567890123456789b\n";
private final static String INPUT_STRING_NO3 = "01234567890123456789012345678901234567890123456789012345678901234567890123456789h\n";
private final static String SHA256_CHECKSUM = "{SHA-256}c6ab15af7842d23d3c06c138b53a7d09c5e351a79c4eb3c8ca8d65e5ce8900ab";
private final static String SHA1_CHECKSUM = "{SHA-1}49e4b2f4292b63e88597c127d11bc2cc0f2ca0ff";
private final static String MD5_CHECKSUM = "{MD5}d141a8eeaf6bba779d1d1dc5102a81c5";
private final static String ZERO_PADDED_MD5_CHECKSUM = "{MD5}0e51dfa74b87f19dd5e0124d6a2195e3";
private final static String ZERO_PADDED_SHA256_CHECKSUM = "{SHA-256}08b5ae0c7d7d45d8ed406d7c3c7da695b81187903694314d97f8a37752a6b241";
private static final String MD5 = "MD5";
private static final String SHA_256 = "SHA-256";
private final static String WRONG_CHECKSUM = "somethingObviouslyWrong";
private static InputStream inputStream;
private InputStream inputStream2;
@Test
public void check_SHA256() throws Exception {
Assert.assertTrue(DigestHelper.check(SHA256_CHECKSUM, inputStream));
}
@Test
public void check_SHA1() throws Exception {
Assert.assertTrue(DigestHelper.check(SHA1_CHECKSUM, inputStream));
}
@Test
public void check_MD5() throws Exception {
Assert.assertTrue(DigestHelper.check(MD5_CHECKSUM, inputStream));
}
@Test
public void check_invalidString() throws Exception {
Assert.assertFalse(DigestHelper.check(WRONG_CHECKSUM, inputStream));
}
@Test
public void testDigestSHA256() throws Exception {
String result = DigestHelper.digest(SHA_256, inputStream).toString();
Assert.assertEquals(SHA256_CHECKSUM, result);
}
@Test
public void testDigestSHA1() throws Exception {
String result = DigestHelper.digest("SHA-1", inputStream).toString();
Assert.assertEquals(SHA1_CHECKSUM, result);
}
@Test
public void testDigestMD5() throws Exception {
String result = DigestHelper.digest(MD5, inputStream).toString();
Assert.assertEquals(MD5_CHECKSUM, result);
}
@Test
public void testZeroPaddedDigestMD5() throws Exception {
inputStream2 = new StringInputStream(INPUT_STRING_NO2);
String result = DigestHelper.digest(MD5, inputStream2).toString();
Assert.assertEquals(ZERO_PADDED_MD5_CHECKSUM, result);
}
@Test
public void testZeroPaddedDigestSHA256() throws Exception {
inputStream2 = new StringInputStream(INPUT_STRING_NO3);
String result = DigestHelper.digest(SHA_256, inputStream2).toString();
Assert.assertEquals(ZERO_PADDED_SHA256_CHECKSUM, result);
}
@BeforeClass
public static void init() throws UnsupportedEncodingException {
inputStream = new StringInputStream(INPUT_STRING);
}
@Before
public void reset() throws IOException {
inputStream.reset();
}
}
//Generated with love by TestMe :) Please report issues and submit feature requests at: http://weirddev.com/forum#!/testme