From ed7811a9a2589395fcfe8341b870ef14215e008f Mon Sep 17 00:00:00 2001 From: dahn Date: Wed, 11 Oct 2017 11:49:06 +0200 Subject: [PATCH] CLOUDSTACK-10046 checksum validation for any java supported Digests-type (#2246) * CLOUDSTACK-10046 digest helper for calculating checksums * CLOUDSTACK-10046 cleanup unused checksum code * CLOUDSTACK-10046 padding method proof of concept * CLOUDSTACK-10046 only compare checksums if old value is valid * Adding positive and negative tests for md5, sha-1 and sha-256, for xen, vmware and kvm hypervisors. KVM Results: Negative Test Passed - Exception Occurred Under template download ['Traceback (most recent call last):\n', ' File "/Users/bstoyanov/Documents/sb2/cloudstack/test/integration/smoke/test_templates.py", line 189, in test_02_1_create_template_with_checksum_sha1_negative\n self.download(self.apiclient, template.id)\n', ' File "/Users/bstoyanov/Documents/sb2/cloudstack/test/integration/smoke/test_templates.py", line 260, in download\n template.status)\n', 'Exception: Failed to download template: status - Failed post download script: checksum "{sha-1}bf580a13f791d86acf3449a7b457a91a14389264" didn\'t match the given value, "{sha-1}someInvalidValue"\n'] === TestName: test_02_1_create_template_with_checksum_sha1_negative | Status : SUCCESS === === TestName: test_02_create_template_with_checksum_sha1 | Status : SUCCESS ===. Negative Test Passed - Exception Occurred Under template download ['Traceback (most recent call last):\n', ' File "/Users/bstoyanov/Documents/sb2/cloudstack/test/integration/smoke/test_templates.py", line 203, in test_03_1_create_template_with_checksum_sha256_negative\n self.download(self.apiclient, template.id)\n', ' File "/Users/bstoyanov/Documents/sb2/cloudstack/test/integration/smoke/test_templates.py", line 260, in download\n template.status)\n', 'Exception: Failed to download template: status - Failed post download script: checksum "{SHA-256}efc03633f2b8f5db08acbcc5dc1be9028572dfd8f1c6c8ea663f0ef94b458c5" didn\'t match the given value, "{SHA-256}someInvalidValue"\n'] === TestName: test_03_1_create_template_with_checksum_sha256_negative | Status : SUCCESS === === TestName: test_03_create_template_with_checksum_sha256 | Status : SUCCESS === Negative Test Passed - Exception Occurred Under template download ['Traceback (most recent call last):\n', ' File "/Users/bstoyanov/Documents/sb2/cloudstack/test/integration/smoke/test_templates.py", line 217, in test_04_1_create_template_with_checksum_md5_negative\n self.download(self.apiclient, template.id)\n', ' File "/Users/bstoyanov/Documents/sb2/cloudstack/test/integration/smoke/test_templates.py", line 260, in download\n template.status)\n', 'Exception: Failed to download template: status - Failed post download script: checksum "{md5}ada77653dcf1e59495a9e1ac670ad95f" didn\'t match the given value, "{md5}someInvalidValue"\n'] === TestName: test_04_1_create_template_with_checksum_md5_negative | Status : SUCCESS === === TestName: test_04_create_template_with_checksum_md5 | Status : SUCCESS === * CLOUDSTACK-10046 digest helper for calculating checksums * CLOUDSTACK-10046 cleanup unused checksum code * CLOUDSTACK-10046 padding method proof of concept * CLOUDSTACK-10046 only compare checksums if old value is valid * Adding positive and negative tests for md5, sha-1 and sha-256, for xen, vmware and kvm hypervisors. KVM Results: Negative Test Passed - Exception Occurred Under template download ['Traceback (most recent call last):\n', ' File "/Users/bstoyanov/Documents/sb2/cloudstack/test/integration/smoke/test_templates.py", line 189, in test_02_1_create_template_with_checksum_sha1_negative\n self.download(self.apiclient, template.id)\n', ' File "/Users/bstoyanov/Documents/sb2/cloudstack/test/integration/smoke/test_templates.py", line 260, in download\n template.status)\n', 'Exception: Failed to download template: status - Failed post download script: checksum "{sha-1}bf580a13f791d86acf3449a7b457a91a14389264" didn\'t match the given value, "{sha-1}someInvalidValue"\n'] === TestName: test_02_1_create_template_with_checksum_sha1_negative | Status : SUCCESS === === TestName: test_02_create_template_with_checksum_sha1 | Status : SUCCESS ===. Negative Test Passed - Exception Occurred Under template download ['Traceback (most recent call last):\n', ' File "/Users/bstoyanov/Documents/sb2/cloudstack/test/integration/smoke/test_templates.py", line 203, in test_03_1_create_template_with_checksum_sha256_negative\n self.download(self.apiclient, template.id)\n', ' File "/Users/bstoyanov/Documents/sb2/cloudstack/test/integration/smoke/test_templates.py", line 260, in download\n template.status)\n', 'Exception: Failed to download template: status - Failed post download script: checksum "{SHA-256}efc03633f2b8f5db08acbcc5dc1be9028572dfd8f1c6c8ea663f0ef94b458c5" didn\'t match the given value, "{SHA-256}someInvalidValue"\n'] === TestName: test_03_1_create_template_with_checksum_sha256_negative | Status : SUCCESS === === TestName: test_03_create_template_with_checksum_sha256 | Status : SUCCESS === Negative Test Passed - Exception Occurred Under template download ['Traceback (most recent call last):\n', ' File "/Users/bstoyanov/Documents/sb2/cloudstack/test/integration/smoke/test_templates.py", line 217, in test_04_1_create_template_with_checksum_md5_negative\n self.download(self.apiclient, template.id)\n', ' File "/Users/bstoyanov/Documents/sb2/cloudstack/test/integration/smoke/test_templates.py", line 260, in download\n template.status)\n', 'Exception: Failed to download template: status - Failed post download script: checksum "{md5}ada77653dcf1e59495a9e1ac670ad95f" didn\'t match the given value, "{md5}someInvalidValue"\n'] === TestName: test_04_1_create_template_with_checksum_md5_negative | Status : SUCCESS === === TestName: test_04_create_template_with_checksum_md5 | Status : SUCCESS === * Adding additional test with no checksum added when registering template Result: test_05_create_template_with_no_checksum (integration.smoke.test_templates.TestCreateTemplateWithChecksum) ... === TestName: test_05_create_template_with_no_checksum | Status : SUCCESS === ok ---------------------------------------------------------------------- Ran 1 test in 42.320s OK * Fixing negative tests exception handling * Adding tests for ISO checksum validation and fixing a zero prefix failure test in templates * CLOUDSTACK-10046 padding * CLOUDSTACK-10046 usability additions * yet another IDE artifact hindering checkstyle --- .../org/apache/cloudstack/api/APICommand.java | 5 +- .../api/AbstractGetUploadParamsCmd.java | 8 +- .../apache/cloudstack/api/ApiConstants.java | 6 + .../api/command/user/iso/RegisterIsoCmd.java | 2 +- .../user/template/RegisterTemplateCmd.java | 2 +- .../command/user/volume/UploadVolumeCmd.java | 2 +- .../agent/api/ComputeChecksumCommand.java | 14 +- .../com/cloud/template/TemplateManager.java | 2 +- scripts/installer/createtmplt.sh | 1 + scripts/installer/createvolume.sh | 1 + scripts/storage/qcow2/createtmplt.sh | 24 +-- scripts/storage/qcow2/createvolume.sh | 24 +-- scripts/storage/secondary/createtmplt.sh | 34 +-- scripts/storage/secondary/createvolume.sh | 1 + .../cloud/template/TemplateManagerImpl.java | 4 +- server/src/com/cloud/test/DatabaseConfig.java | 38 ++-- .../resource/NfsSecondaryStorageResource.java | 118 +++++------ .../storage/template/DownloadManagerImpl.java | 76 +++---- test/integration/smoke/test_iso.py | 166 ++++++++++++++- test/integration/smoke/test_templates.py | 199 ++++++++++++++++++ .../utils/security/ChecksumValue.java | 86 ++++++++ .../utils/security/DigestHelper.java | 96 +++++++++ .../utils/security/DigestHelperTest.java | 102 +++++++++ 23 files changed, 774 insertions(+), 237 deletions(-) create mode 100644 utils/src/main/java/org/apache/cloudstack/utils/security/ChecksumValue.java create mode 100644 utils/src/main/java/org/apache/cloudstack/utils/security/DigestHelper.java create mode 100644 utils/src/test/java/org/apache/cloudstack/utils/security/DigestHelperTest.java diff --git a/api/src/org/apache/cloudstack/api/APICommand.java b/api/src/org/apache/cloudstack/api/APICommand.java index d451e4b10a3..c559be08116 100644 --- a/api/src/org/apache/cloudstack/api/APICommand.java +++ b/api/src/org/apache/cloudstack/api/APICommand.java @@ -16,8 +16,6 @@ // under the License. package org.apache.cloudstack.api; -import static java.lang.annotation.ElementType.TYPE; - import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; @@ -25,9 +23,12 @@ import java.lang.annotation.Target; import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.api.ResponseObject.ResponseView; +import static java.lang.annotation.ElementType.TYPE; + @Retention(RetentionPolicy.RUNTIME) @Target({TYPE}) public @interface APICommand { + Class responseObject(); String name() default ""; diff --git a/api/src/org/apache/cloudstack/api/AbstractGetUploadParamsCmd.java b/api/src/org/apache/cloudstack/api/AbstractGetUploadParamsCmd.java index df63d747158..c82f4789367 100644 --- a/api/src/org/apache/cloudstack/api/AbstractGetUploadParamsCmd.java +++ b/api/src/org/apache/cloudstack/api/AbstractGetUploadParamsCmd.java @@ -18,15 +18,15 @@ */ package org.apache.cloudstack.api; +import java.net.URL; +import java.util.UUID; + import org.apache.cloudstack.api.response.DomainResponse; import org.apache.cloudstack.api.response.GetUploadParamsResponse; import org.apache.cloudstack.api.response.ProjectResponse; import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.log4j.Logger; -import java.net.URL; -import java.util.UUID; - public abstract class AbstractGetUploadParamsCmd extends BaseCmd { public static final Logger s_logger = Logger.getLogger(AbstractGetUploadParamsCmd.class.getName()); @@ -42,7 +42,7 @@ public abstract class AbstractGetUploadParamsCmd extends BaseCmd { + "to be hosted on") private Long zoneId; - @Parameter(name = ApiConstants.CHECKSUM, type = CommandType.STRING, description = "the MD5 checksum value of this volume/template") + @Parameter(name = ApiConstants.CHECKSUM, type = CommandType.STRING, description = "the checksum value of this volume/template " + ApiConstants.CHECKSUM_PARAMETER_PREFIX_DESCRIPTION) private String checksum; @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, description = "an optional accountName. Must be used with domainId.") diff --git a/api/src/org/apache/cloudstack/api/ApiConstants.java b/api/src/org/apache/cloudstack/api/ApiConstants.java index 0a8a112d737..2300e680df6 100644 --- a/api/src/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/org/apache/cloudstack/api/ApiConstants.java @@ -673,6 +673,12 @@ public class ApiConstants { public static final String ZONE_ID_LIST = "zoneids"; public static final String DESTINATION_ZONE_ID_LIST = "destzoneids"; public static final String ADMIN = "admin"; + public static final String CHECKSUM_PARAMETER_PREFIX_DESCRIPTION = "The parameter containing the checksum will be considered a MD5sum if it is not prefixed\n" + + " and just a plain ascii/utf8 representation of a hexadecimal string. If it is required to\n" + + " use another algorithm the hexadecimal string is to be prefixed with a string of the form,\n" + + " \"{}\", not including the double quotes. In this is the exact string\n" + + " representing the java supported algorithm, i.e. MD5 or SHA-256. Note that java does not\n" + + " contain an algorithm called SHA256 or one called sha-256, only SHA-256."; public enum HostDetails { all, capacity, events, stats, min; diff --git a/api/src/org/apache/cloudstack/api/command/user/iso/RegisterIsoCmd.java b/api/src/org/apache/cloudstack/api/command/user/iso/RegisterIsoCmd.java index 599aac1fd03..3112287fb9c 100644 --- a/api/src/org/apache/cloudstack/api/command/user/iso/RegisterIsoCmd.java +++ b/api/src/org/apache/cloudstack/api/command/user/iso/RegisterIsoCmd.java @@ -94,7 +94,7 @@ public class RegisterIsoCmd extends BaseCmd { @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, description = "an optional account name. Must be used with domainId.") private String accountName; - @Parameter(name = ApiConstants.CHECKSUM, type = CommandType.STRING, description = "the MD5 checksum value of this ISO") + @Parameter(name = ApiConstants.CHECKSUM, type = CommandType.STRING, description = "the checksum value of this ISO. " + ApiConstants.CHECKSUM_PARAMETER_PREFIX_DESCRIPTION) private String checksum; @Parameter(name = ApiConstants.PROJECT_ID, type = CommandType.UUID, entityType = ProjectResponse.class, description = "Register ISO for the project") diff --git a/api/src/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java b/api/src/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java index 9e57574ba21..2bd7b2d38d1 100644 --- a/api/src/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java +++ b/api/src/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java @@ -122,7 +122,7 @@ public class RegisterTemplateCmd extends BaseCmd { @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, description = "an optional accountName. Must be used with domainId.") private String accountName; - @Parameter(name = ApiConstants.CHECKSUM, type = CommandType.STRING, description = "the MD5 checksum value of this template") + @Parameter(name = ApiConstants.CHECKSUM, type = CommandType.STRING, description = "the checksum value of this template. " + ApiConstants.CHECKSUM_PARAMETER_PREFIX_DESCRIPTION) private String checksum; @Parameter(name = ApiConstants.TEMPLATE_TAG, type = CommandType.STRING, description = "the tag for this template.") diff --git a/api/src/org/apache/cloudstack/api/command/user/volume/UploadVolumeCmd.java b/api/src/org/apache/cloudstack/api/command/user/volume/UploadVolumeCmd.java index 21749616753..a48a89b79b4 100644 --- a/api/src/org/apache/cloudstack/api/command/user/volume/UploadVolumeCmd.java +++ b/api/src/org/apache/cloudstack/api/command/user/volume/UploadVolumeCmd.java @@ -82,7 +82,7 @@ public class UploadVolumeCmd extends BaseAsyncCmd { @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, description = "an optional accountName. Must be used with domainId.") private String accountName; - @Parameter(name = ApiConstants.CHECKSUM, type = CommandType.STRING, description = "the MD5 checksum value of this volume") + @Parameter(name = ApiConstants.CHECKSUM, type = CommandType.STRING, description = "the checksum value of this volume. " + ApiConstants.CHECKSUM_PARAMETER_PREFIX_DESCRIPTION) private String checksum; @Parameter(name = ApiConstants.IMAGE_STORE_UUID, type = CommandType.STRING, description = "Image store uuid") diff --git a/core/src/com/cloud/agent/api/ComputeChecksumCommand.java b/core/src/com/cloud/agent/api/ComputeChecksumCommand.java index 5c8c929ce6f..fc7c55d6f52 100644 --- a/core/src/com/cloud/agent/api/ComputeChecksumCommand.java +++ b/core/src/com/cloud/agent/api/ComputeChecksumCommand.java @@ -25,6 +25,7 @@ import com.cloud.agent.api.to.DataStoreTO; public class ComputeChecksumCommand extends SsCommand { private DataStoreTO store; private String templatePath; + private String algorithm = "MD5"; public ComputeChecksumCommand() { super(); @@ -35,6 +36,11 @@ public class ComputeChecksumCommand extends SsCommand { this.setStore(store); } + public ComputeChecksumCommand(DataStoreTO store, String templatePath, String algorithm) { + this(store,templatePath); + this.algorithm = algorithm; + } + public String getTemplatePath() { return templatePath; } @@ -43,8 +49,12 @@ public class ComputeChecksumCommand extends SsCommand { return store; } - public void setStore(DataStoreTO store) { - this.store = store; + + public String getAlgorithm() { + return algorithm; } + void setStore(DataStoreTO store) { + this.store = store; + } } diff --git a/engine/components-api/src/com/cloud/template/TemplateManager.java b/engine/components-api/src/com/cloud/template/TemplateManager.java index 5b2d64ad5a3..68c3b585692 100644 --- a/engine/components-api/src/com/cloud/template/TemplateManager.java +++ b/engine/components-api/src/com/cloud/template/TemplateManager.java @@ -115,7 +115,7 @@ public interface TemplateManager { DataStore getImageStore(String storeUuid, Long zoneId); - String getChecksum(DataStore store, String templatePath); + String getChecksum(DataStore store, String templatePath, String algorithm); List getImageStoreByTemplate(long templateId, Long zoneId); diff --git a/scripts/installer/createtmplt.sh b/scripts/installer/createtmplt.sh index 67b25c3b923..c187c5fcb12 100755 --- a/scripts/installer/createtmplt.sh +++ b/scripts/installer/createtmplt.sh @@ -39,6 +39,7 @@ fi verify_cksum() { digestalgo="" +# NOTE this will only work with 0-padded checksums case ${#1} in 32) digestalgo="md5sum" ;; 40) digestalgo="sha1sum" ;; diff --git a/scripts/installer/createvolume.sh b/scripts/installer/createvolume.sh index 00ee5e5e773..c7f11dc237c 100755 --- a/scripts/installer/createvolume.sh +++ b/scripts/installer/createvolume.sh @@ -40,6 +40,7 @@ fi verify_cksum() { digestalgo="" +# NOTE this will only work with 0-padded checksums case ${#1} in 32) digestalgo="md5sum" ;; 40) digestalgo="sha1sum" ;; diff --git a/scripts/storage/qcow2/createtmplt.sh b/scripts/storage/qcow2/createtmplt.sh index 11645250e55..b05550c14bb 100755 --- a/scripts/storage/qcow2/createtmplt.sh +++ b/scripts/storage/qcow2/createtmplt.sh @@ -21,7 +21,7 @@ # createtmplt.sh -- install a template usage() { - printf "Usage: %s: -t -n -f -s -c -d -h [-u]\n" $(basename $0) >&2 + printf "Usage: %s: -t -n -f -s -c -d -h [-u]\n" $(basename $0) >&2 } @@ -37,27 +37,6 @@ then fi fi - -verify_cksum() { - digestalgo="" - case ${#1} in - 32) digestalgo="md5sum" ;; - 40) digestalgo="sha1sum" ;; - 56) digestalgo="sha224sum" ;; - 64) digestalgo="sha256sum" ;; - 96) digestalgo="sha384sum" ;; - 128) digestalgo="sha512sum" ;; - *) echo "Please provide valid cheksum" ; exit 3 ;; - esac - echo "$1 $2" | $digestalgo -c --status - #printf "$1\t$2" | $digestalgo -c --status - if [ $? -gt 0 ] - then - printf "Checksum failed, not proceeding with install\n" - exit 3 - fi -} - untar() { local ft=$(file $1| awk -F" " '{print $2}') local basedir=$(dirname $1) @@ -166,7 +145,6 @@ do tmpltimg="$OPTARG" ;; s) sflag=1 - sflag=1 ;; c) cflag=1 snapshotName="$OPTARG" diff --git a/scripts/storage/qcow2/createvolume.sh b/scripts/storage/qcow2/createvolume.sh index 91a76323f15..033cc91b108 100755 --- a/scripts/storage/qcow2/createvolume.sh +++ b/scripts/storage/qcow2/createvolume.sh @@ -22,7 +22,7 @@ # createvol.sh -- install a volume usage() { - printf "Usage: %s: -t -n -f -s -c -d -h [-u]\n" $(basename $0) >&2 + printf "Usage: %s: -t -n -f -s -c -d -h [-u]\n" $(basename $0) >&2 } @@ -38,27 +38,6 @@ then fi fi - -verify_cksum() { - digestalgo="" - case ${#1} in - 32) digestalgo="md5sum" ;; - 40) digestalgo="sha1sum" ;; - 56) digestalgo="sha224sum" ;; - 64) digestalgo="sha256sum" ;; - 96) digestalgo="sha384sum" ;; - 128) digestalgo="sha512sum" ;; - *) echo "Please provide valid cheksum" ; exit 3 ;; - esac - echo "$1 $2" | $digestalgo -c --status - #printf "$1\t$2" | $digestalgo -c --status - if [ $? -gt 0 ] - then - printf "Checksum failed, not proceeding with install\n" - exit 3 - fi -} - untar() { local ft=$(file $1| awk -F" " '{print $2}') local basedir=$(dirname $1) @@ -167,7 +146,6 @@ do volimg="$OPTARG" ;; s) sflag=1 - sflag=1 ;; c) cflag=1 snapshotName="$OPTARG" diff --git a/scripts/storage/secondary/createtmplt.sh b/scripts/storage/secondary/createtmplt.sh index acc4ef5c928..4e8db4633f0 100755 --- a/scripts/storage/secondary/createtmplt.sh +++ b/scripts/storage/secondary/createtmplt.sh @@ -22,7 +22,7 @@ # createtmplt.sh -- install a template usage() { - printf "Usage: %s: -t -n -f -c -d -h [-u] [-v]\n" $(basename $0) >&2 + printf "Usage: %s: -t -n -f -d -h [-u] [-v]\n" $(basename $0) >&2 } @@ -39,26 +39,6 @@ rollback_if_needed() { fi } -verify_cksum() { - digestalgo="" - case ${#1} in - 32) digestalgo="md5sum" ;; - 40) digestalgo="sha1sum" ;; - 56) digestalgo="sha224sum" ;; - 64) digestalgo="sha256sum" ;; - 96) digestalgo="sha384sum" ;; - 128) digestalgo="sha512sum" ;; - *) echo "Please provide valid cheksum" ; exit 3 ;; - esac - echo "$1 $2" | $digestalgo -c --status - #printf "$1\t$2" | $digestalgo -c --status - if [ $? -gt 0 ] - then - printf "Checksum failed, not proceeding with install\n" - exit 3 - fi -} - untar() { local ft=$(file $1| awk -F" " '{print $2}') case $ft in @@ -138,9 +118,8 @@ hflag= hvm=false cleanup=false dflag= -cflag= -while getopts 'vuht:n:f:s:c:d:S:' OPTION +while getopts 'vuht:n:f:s:d:S:' OPTION do case $OPTION in t) tflag=1 @@ -154,9 +133,6 @@ do ;; s) sflag=1 ;; - c) cflag=1 - cksum="$OPTARG" - ;; d) dflag=1 descr="$OPTARG" ;; @@ -200,10 +176,6 @@ then exit 3 fi -if [ -n "$cksum" ] -then - verify_cksum $cksum $tmpltimg -fi [ -n "$verbose" ] && is_compressed $tmpltimg tmpltimg2=$(uncompress $tmpltimg) rollback_if_needed $tmpltfs $? "failed to uncompress $tmpltimg\n" @@ -236,6 +208,8 @@ echo -n "" > /$tmpltfs/template.properties today=$(date '+%m_%d_%Y') echo "filename=$tmpltname" > /$tmpltfs/template.properties echo "description=$descr" >> /$tmpltfs/template.properties +# we need to rethink this property as it might get changed after download due to decompression +# option is to recalcutate it here echo "checksum=$cksum" >> /$tmpltfs/template.properties echo "hvm=$hvm" >> /$tmpltfs/template.properties echo "size=$imgsize" >> /$tmpltfs/template.properties diff --git a/scripts/storage/secondary/createvolume.sh b/scripts/storage/secondary/createvolume.sh index c7836dcccb5..12f73ebc109 100755 --- a/scripts/storage/secondary/createvolume.sh +++ b/scripts/storage/secondary/createvolume.sh @@ -41,6 +41,7 @@ fi verify_cksum() { digestalgo="" +# NOTE this will only work with 0-padded checksums case ${#1} in 32) digestalgo="md5sum" ;; 40) digestalgo="sha1sum" ;; diff --git a/server/src/com/cloud/template/TemplateManagerImpl.java b/server/src/com/cloud/template/TemplateManagerImpl.java index 86f687ce137..f6494c3b77e 100644 --- a/server/src/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/com/cloud/template/TemplateManagerImpl.java @@ -673,9 +673,9 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, } @Override - public String getChecksum(DataStore store, String templatePath) { + public String getChecksum(DataStore store, String templatePath, String algorithm) { EndPoint ep = _epSelector.select(store); - ComputeChecksumCommand cmd = new ComputeChecksumCommand(store.getTO(), templatePath); + ComputeChecksumCommand cmd = new ComputeChecksumCommand(store.getTO(), templatePath, algorithm); Answer answer = null; if (ep == null) { String errMsg = "No remote endpoint to send command, check if host or ssvm is down?"; diff --git a/server/src/com/cloud/test/DatabaseConfig.java b/server/src/com/cloud/test/DatabaseConfig.java index a27f6710631..7240374bfa7 100644 --- a/server/src/com/cloud/test/DatabaseConfig.java +++ b/server/src/com/cloud/test/DatabaseConfig.java @@ -18,9 +18,7 @@ package com.cloud.test; import java.io.File; import java.io.IOException; -import java.math.BigInteger; import java.net.URISyntaxException; -import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.sql.Date; import java.sql.PreparedStatement; @@ -39,21 +37,12 @@ import javax.xml.parsers.ParserConfigurationException; import javax.xml.parsers.SAXParser; import javax.xml.parsers.SAXParserFactory; -import org.apache.log4j.Logger; -import org.apache.log4j.xml.DOMConfigurator; -import org.w3c.dom.Document; -import org.w3c.dom.Node; -import org.w3c.dom.NodeList; -import org.xml.sax.Attributes; -import org.xml.sax.SAXException; -import org.xml.sax.helpers.DefaultHandler; - import com.cloud.host.Status; import com.cloud.service.ServiceOfferingVO; import com.cloud.service.dao.ServiceOfferingDaoImpl; import com.cloud.storage.DiskOfferingVO; -import com.cloud.storage.dao.DiskOfferingDaoImpl; import com.cloud.storage.Storage.ProvisioningType; +import com.cloud.storage.dao.DiskOfferingDaoImpl; import com.cloud.utils.PropertiesUtil; import com.cloud.utils.component.ComponentContext; import com.cloud.utils.db.DB; @@ -62,6 +51,15 @@ import com.cloud.utils.db.TransactionCallbackWithExceptionNoReturn; import com.cloud.utils.db.TransactionLegacy; import com.cloud.utils.db.TransactionStatus; import com.cloud.utils.net.NfsUtils; +import org.apache.cloudstack.utils.security.DigestHelper; +import org.apache.log4j.Logger; +import org.apache.log4j.xml.DOMConfigurator; +import org.w3c.dom.Document; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; +import org.xml.sax.Attributes; +import org.xml.sax.SAXException; +import org.xml.sax.helpers.DefaultHandler; public class DatabaseConfig { private static final Logger s_logger = Logger.getLogger(DatabaseConfig.class.getName()); @@ -1169,22 +1167,14 @@ public class DatabaseConfig { printError("An email address for each user is required."); } - MessageDigest md5 = null; + String algorithm = "MD5"; + String pwDigest; try { - md5 = MessageDigest.getInstance("MD5"); + pwDigest = DigestHelper.getPaddedDigest(algorithm, password); } catch (NoSuchAlgorithmException e) { s_logger.error("error saving user", e); return; } - md5.reset(); - BigInteger pwInt = new BigInteger(1, md5.digest(password.getBytes())); - String pwStr = pwInt.toString(16); - int padding = 32 - pwStr.length(); - StringBuffer sb = new StringBuffer(); - for (int i = 0; i < padding; i++) { - sb.append('0'); // make sure the MD5 password is 32 digits long - } - sb.append(pwStr); // create an account for the admin user first final String insertAdminAccount = "INSERT INTO `cloud`.`account` (id, account_name, type, domain_id) VALUES (?, ?, '1', '1')"; @@ -1206,7 +1196,7 @@ public class DatabaseConfig { PreparedStatement stmt = txn.prepareAutoCloseStatement(insertUser); stmt.setLong(1, id); stmt.setString(2, username); - stmt.setString(3, sb.toString()); + stmt.setString(3, pwDigest); stmt.setString(4, firstname); stmt.setString(5, lastname); stmt.setString(6, email); diff --git a/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java b/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java index 68569ea7f9a..37cb72830b4 100644 --- a/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java +++ b/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java @@ -16,12 +16,6 @@ // under the License. package org.apache.cloudstack.storage.resource; -import static com.cloud.utils.StringUtils.join; -import static com.cloud.utils.storage.S3.S3Utils.putFile; -import static java.lang.String.format; -import static java.util.Arrays.asList; -import static org.apache.commons.lang.StringUtils.substringAfterLast; - import java.io.BufferedReader; import java.io.BufferedWriter; import java.io.File; @@ -32,11 +26,9 @@ import java.io.FileWriter; import java.io.IOException; import java.io.InputStream; import java.io.UnsupportedEncodingException; -import java.math.BigInteger; import java.net.InetAddress; import java.net.URI; import java.net.UnknownHostException; -import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.ArrayList; import java.util.HashMap; @@ -46,41 +38,6 @@ import java.util.UUID; import javax.naming.ConfigurationException; -import org.apache.cloudstack.framework.security.keystore.KeystoreManager; -import org.apache.cloudstack.storage.command.CopyCmdAnswer; -import org.apache.cloudstack.storage.command.CopyCommand; -import org.apache.cloudstack.storage.command.DeleteCommand; -import org.apache.cloudstack.storage.command.DownloadCommand; -import org.apache.cloudstack.storage.command.DownloadProgressCommand; -import org.apache.cloudstack.storage.command.TemplateOrVolumePostUploadCommand; -import org.apache.cloudstack.storage.command.UploadStatusAnswer; -import org.apache.cloudstack.storage.command.UploadStatusAnswer.UploadStatus; -import org.apache.cloudstack.storage.command.UploadStatusCommand; -import org.apache.cloudstack.storage.template.DownloadManager; -import org.apache.cloudstack.storage.template.DownloadManagerImpl; -import org.apache.cloudstack.storage.template.DownloadManagerImpl.ZfsPathParser; -import org.apache.cloudstack.storage.template.UploadEntity; -import org.apache.cloudstack.storage.template.UploadManager; -import org.apache.cloudstack.storage.template.UploadManagerImpl; -import org.apache.cloudstack.storage.to.SnapshotObjectTO; -import org.apache.cloudstack.storage.to.TemplateObjectTO; -import org.apache.cloudstack.storage.to.VolumeObjectTO; -import org.apache.cloudstack.utils.imagestore.ImageStoreUtil; -import org.apache.commons.codec.digest.DigestUtils; -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.FilenameUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.http.HttpEntity; -import org.apache.http.HttpResponse; -import org.apache.http.NameValuePair; -import org.apache.http.client.HttpClient; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.utils.URLEncodedUtils; -import org.apache.http.impl.client.DefaultHttpClient; -import org.apache.log4j.Logger; -import org.joda.time.DateTime; -import org.joda.time.format.ISODateTimeFormat; - import com.amazonaws.services.s3.model.S3ObjectSummary; import com.cloud.agent.api.Answer; import com.cloud.agent.api.CheckHealthAnswer; @@ -148,7 +105,6 @@ import com.cloud.utils.storage.S3.S3Utils; import com.cloud.vm.SecondaryStorageVm; import com.google.gson.Gson; import com.google.gson.GsonBuilder; - import io.netty.bootstrap.ServerBootstrap; import io.netty.channel.Channel; import io.netty.channel.ChannelInitializer; @@ -162,6 +118,47 @@ import io.netty.handler.codec.http.HttpRequestDecoder; import io.netty.handler.codec.http.HttpResponseEncoder; import io.netty.handler.logging.LogLevel; import io.netty.handler.logging.LoggingHandler; +import org.apache.cloudstack.framework.security.keystore.KeystoreManager; +import org.apache.cloudstack.storage.command.CopyCmdAnswer; +import org.apache.cloudstack.storage.command.CopyCommand; +import org.apache.cloudstack.storage.command.DeleteCommand; +import org.apache.cloudstack.storage.command.DownloadCommand; +import org.apache.cloudstack.storage.command.DownloadProgressCommand; +import org.apache.cloudstack.storage.command.TemplateOrVolumePostUploadCommand; +import org.apache.cloudstack.storage.command.UploadStatusAnswer; +import org.apache.cloudstack.storage.command.UploadStatusAnswer.UploadStatus; +import org.apache.cloudstack.storage.command.UploadStatusCommand; +import org.apache.cloudstack.storage.template.DownloadManager; +import org.apache.cloudstack.storage.template.DownloadManagerImpl; +import org.apache.cloudstack.storage.template.DownloadManagerImpl.ZfsPathParser; +import org.apache.cloudstack.storage.template.UploadEntity; +import org.apache.cloudstack.storage.template.UploadManager; +import org.apache.cloudstack.storage.template.UploadManagerImpl; +import org.apache.cloudstack.storage.to.SnapshotObjectTO; +import org.apache.cloudstack.storage.to.TemplateObjectTO; +import org.apache.cloudstack.storage.to.VolumeObjectTO; +import org.apache.cloudstack.utils.imagestore.ImageStoreUtil; +import org.apache.cloudstack.utils.security.DigestHelper; +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.http.HttpEntity; +import org.apache.http.HttpResponse; +import org.apache.http.NameValuePair; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.utils.URLEncodedUtils; +import org.apache.http.impl.client.DefaultHttpClient; +import org.apache.log4j.Logger; +import org.joda.time.DateTime; +import org.joda.time.format.ISODateTimeFormat; + +import static com.cloud.utils.StringUtils.join; +import static com.cloud.utils.storage.S3.S3Utils.putFile; +import static java.lang.String.format; +import static java.util.Arrays.asList; +import static org.apache.commons.lang.StringUtils.substringAfterLast; public class NfsSecondaryStorageResource extends ServerResourceBase implements SecondaryStorageResource { @@ -1316,46 +1313,24 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S parent += File.separator; } String absoluteTemplatePath = parent + relativeTemplatePath; - MessageDigest digest; - String checksum = null; + String algorithm = cmd.getAlgorithm(); File f = new File(absoluteTemplatePath); - InputStream is = null; - byte[] buffer = new byte[8192]; - int read = 0; if (s_logger.isDebugEnabled()) { s_logger.debug("parent path " + parent + " relative template path " + relativeTemplatePath); } + String checksum = null; - try { - digest = MessageDigest.getInstance("MD5"); - is = new FileInputStream(f); - while ((read = is.read(buffer)) > 0) { - digest.update(buffer, 0, read); - } - byte[] md5sum = digest.digest(); - BigInteger bigInt = new BigInteger(1, md5sum); - checksum = bigInt.toString(16); + try (InputStream is = new FileInputStream(f);){ + checksum = DigestHelper.digest(algorithm, is).toString(); if (s_logger.isDebugEnabled()) { s_logger.debug("Successfully calculated checksum for file " + absoluteTemplatePath + " - " + checksum); } - } catch (IOException e) { - String logMsg = "Unable to process file for MD5 - " + absoluteTemplatePath; + String logMsg = "Unable to process file for " + algorithm + " - " + absoluteTemplatePath; s_logger.error(logMsg); return new Answer(cmd, false, checksum); } catch (NoSuchAlgorithmException e) { return new Answer(cmd, false, checksum); - } finally { - try { - if (is != null) { - is.close(); - } - } catch (IOException e) { - if (s_logger.isDebugEnabled()) { - s_logger.debug("Could not close the file " + absoluteTemplatePath); - } - return new Answer(cmd, false, checksum); - } } return new Answer(cmd, true, checksum); @@ -3054,4 +3029,5 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S } return cmd; } + } diff --git a/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/DownloadManagerImpl.java b/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/DownloadManagerImpl.java index 40a1e1cdde0..833ef090f53 100644 --- a/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/DownloadManagerImpl.java +++ b/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/DownloadManagerImpl.java @@ -21,10 +21,8 @@ import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; -import java.math.BigInteger; import java.net.URI; import java.net.URISyntaxException; -import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.text.SimpleDateFormat; import java.util.ArrayList; @@ -84,6 +82,8 @@ import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.script.OutputInterpreter; import com.cloud.utils.script.Script; import com.cloud.utils.storage.QCOW2Utils; +import org.apache.cloudstack.utils.security.ChecksumValue; +import org.apache.cloudstack.utils.security.DigestHelper; public class DownloadManagerImpl extends ManagerBase implements DownloadManager { private String _name; @@ -315,33 +315,11 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager } } - private String computeCheckSum(File f) { - byte[] buffer = new byte[8192]; - int read = 0; - MessageDigest digest; - String checksum = null; - InputStream is = null; - try { - digest = MessageDigest.getInstance("MD5"); - is = new FileInputStream(f); - while ((read = is.read(buffer)) > 0) { - digest.update(buffer, 0, read); - } - byte[] md5sum = digest.digest(); - BigInteger bigInt = new BigInteger(1, md5sum); - checksum = String.format("%032x", bigInt); - return checksum; + private ChecksumValue computeCheckSum(String algorithm, File f) throws NoSuchAlgorithmException { + try (InputStream is = new FileInputStream(f);) { + return DigestHelper.digest(algorithm, is); } catch (IOException e) { return null; - } catch (NoSuchAlgorithmException e) { - return null; - } finally { - try { - if (is != null) - is.close(); - } catch (IOException e) { - return null; - } } } @@ -357,12 +335,8 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager // The QCOW2 is the only format with a header, // and as such can be easily read. - try { - InputStream inputStream = td.getS3ObjectInputStream(); - + try (InputStream inputStream = td.getS3ObjectInputStream();) { dnld.setTemplatesize(QCOW2Utils.getVirtualSize(inputStream)); - - inputStream.close(); } catch (IOException e) { result = "Couldn't read QCOW2 virtual size. Error: " + e.getMessage(); @@ -398,11 +372,22 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager ResourceType resourceType = dnld.getResourceType(); File originalTemplate = new File(td.getDownloadLocalPath()); - String checkSum = computeCheckSum(originalTemplate); - if (checkSum == null) { + ChecksumValue oldValue = new ChecksumValue(dnld.getChecksum()); + ChecksumValue newValue = null; + try { + newValue = computeCheckSum(oldValue.getAlgorithm(), originalTemplate); + } catch (NoSuchAlgorithmException e) { + return "checksum algorithm not recognised: " + oldValue.getAlgorithm(); + } + if(StringUtils.isNotBlank(dnld.getChecksum()) && ! oldValue.equals(newValue)) { + return "checksum \"" + newValue +"\" didn't match the given value, \"" + oldValue + "\""; + } + String checksum = newValue.getChecksum(); + if (checksum == null) { s_logger.warn("Something wrong happened when try to calculate the checksum of downloaded template!"); } - dnld.setCheckSum(checkSum); + + dnld.setCheckSum(checksum); int imgSizeGigs = (int)Math.ceil(_storage.getSize(td.getDownloadLocalPath()) * 1.0d / (1024 * 1024 * 1024)); imgSizeGigs++; // add one just in case @@ -435,11 +420,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager scr.add("-n", templateFilename); scr.add("-t", resourcePath); - scr.add("-f", td.getDownloadLocalPath()); // this is the temporary - // template file downloaded - if (dnld.getChecksum() != null && dnld.getChecksum().length() > 1) { - scr.add("-c", dnld.getChecksum()); - } + scr.add("-f", td.getDownloadLocalPath()); // this is the temporary template file downloaded scr.add("-u"); // cleanup String result; result = scr.execute(); @@ -707,6 +688,10 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager return new DownloadAnswer("Invalid Name", VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR); } + if(! DigestHelper.isAlgorithmSupported(cmd.getChecksum())) { + return new DownloadAnswer("invalid algorithm: " + cmd.getChecksum(), VMTemplateStorageResourceAssoc.Status.NOT_DOWNLOADED); + } + DataStoreTO dstore = cmd.getDataStore(); String installPathPrefix = cmd.getInstallPath(); // for NFS, we need to get mounted path @@ -865,17 +850,6 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager result.put(tInfo.getTemplateName(), tInfo); s_logger.debug("Added template name: " + tInfo.getTemplateName() + ", path: " + tmplt); } - /* - for (String tmplt : isoTmplts) { - String tmp[]; - tmp = tmplt.split("/"); - String tmpltName = tmp[tmp.length - 2]; - tmplt = tmplt.substring(tmplt.lastIndexOf("iso/")); - TemplateInfo tInfo = new TemplateInfo(tmpltName, tmplt, false); - s_logger.debug("Added iso template name: " + tmpltName + ", path: " + tmplt); - result.put(tmpltName, tInfo); - } - */ return result; } diff --git a/test/integration/smoke/test_iso.py b/test/integration/smoke/test_iso.py index f55f8182108..afd540a4624 100755 --- a/test/integration/smoke/test_iso.py +++ b/test/integration/smoke/test_iso.py @@ -17,8 +17,10 @@ """ BVT tests for Templates ISO """ # Import Local Modules +from marvin.cloudstackException import GetDetailExceptionInfo from marvin.cloudstackTestCase import cloudstackTestCase, unittest -from marvin.cloudstackAPI import listZones, updateIso, extractIso, updateIsoPermissions, copyIso, deleteIso +from marvin.cloudstackAPI import listZones, updateIso, extractIso, updateIsoPermissions, copyIso, deleteIso,\ + registerIso,listOsTypes from marvin.lib.utils import cleanup_resources, random_gen, get_hypervisor_type,validateList from marvin.lib.base import Account, Iso from marvin.lib.common import (get_domain, @@ -606,3 +608,165 @@ class TestISO(cloudstackTestCase): self.get_iso_details("vmware-tools.iso") self.get_iso_details("xs-tools.iso") return + + +class TestCreateISOWithChecksum(cloudstackTestCase): + def setUp(self): + self.testClient = super(TestCreateISOWithChecksum, self).getClsTestClient() + self.apiclient = self.testClient.getApiClient() + self.cleanup = [] + + self.unsupportedHypervisor = False + self.hypervisor = self.testClient.getHypervisorInfo() + if self.hypervisor.lower() in ['lxc']: + # Template creation from root volume is not supported in LXC + self.unsupportedHypervisor = True + return + + # Get Zone, Domain and templates + self.zone = get_zone(self.apiclient, self.testClient.getZoneForTests()) + + # Setup default create iso attributes + self.iso = registerIso.registerIsoCmd() + self.iso.checksum = "{SHA-1}" + "e16f703b5d6cb6dd2c448d956be63fcbee7d79ea" + self.iso.zoneid = self.zone.id + self.iso.name = 'test-tynyCore-iso' + self.iso.displaytext = 'test-tynyCore-iso' + self.iso.url = "http://dl.openvm.eu/cloudstack/iso/TinyCore-8.0.iso" + self.iso.ostypeid = self.getOsType("Other Linux (64-bit)") + self.md5 = "f7fee34a73a7f8e3adb30778c7c32c51" + self.sha256 = "069a22f7cc15b34cd39f6dd61ef0cf99ff47a1a92942772c30f50988746517f7" + + if self.unsupportedHypervisor: + self.skipTest("Skipping test because unsupported hypervisor\ + %s" % self.hypervisor) + return + + def tearDown(self): + try: + # Clean up the created templates + for temp in self.cleanup: + cmd = deleteIso.deleteIsoCmd() + cmd.id = temp.id + cmd.zoneid = self.zone.id + self.apiclient.deleteIso(cmd) + + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) + return + + @attr(tags=["advanced", "smoke"], required_hardware="true") + def test_01_create_iso_with_checksum_sha1(self): + iso = self.registerIso(self.iso) + self.download(self.apiclient, iso.id) + + @attr(tags=["advanced", "smoke"], required_hardware="true") + def test_02_create_iso_with_checksum_sha256(self): + self.iso.checksum = "{SHA-256}" + self.sha256 + iso = self.registerIso(self.iso) + self.download(self.apiclient, iso.id) + + @attr(tags=["advanced", "smoke"], required_hardware="true") + def test_03_create_iso_with_checksum_md5(self): + self.iso.checksum = "{md5}" + self.md5 + iso = self.registerIso(self.iso) + self.download(self.apiclient, iso.id) + + @attr(tags=["advanced", "smoke"], required_hardware="true") + def test_01_1_create_iso_with_checksum_sha1_negative(self): + self.iso.checksum = "{sha-1}" + "someInvalidValue" + iso = self.registerIso(self.iso) + + try: + self.download(self.apiclient, iso.id) + except Exception as e: + print "Negative Test Passed - Exception Occurred Under iso download " \ + "%s" % GetDetailExceptionInfo(e) + else: + self.fail("Negative Test Failed - Exception DID NOT Occurred Under iso download ") + + @attr(tags=["advanced", "smoke"], required_hardware="true") + def test_02_1_create_iso_with_checksum_sha256_negative(self): + self.iso.checksum = "{SHA-256}" + "someInvalidValue" + iso = self.registerIso(self.iso) + + try: + self.download(self.apiclient, iso.id) + except Exception as e: + print "Negative Test Passed - Exception Occurred Under iso download " \ + "%s" % GetDetailExceptionInfo(e) + else: + self.fail("Negative Test Failed - Exception DID NOT Occurred Under iso download ") + + @attr(tags=["advanced", "smoke"], required_hardware="true") + def test_03_1_create_iso_with_checksum_md5_negative(self): + self.iso.checksum = "{md5}" + "someInvalidValue" + iso = self.registerIso(self.iso) + + try: + self.download(self.apiclient, iso.id) + except Exception as e: + print "Negative Test Passed - Exception Occurred Under iso download " \ + "%s" % GetDetailExceptionInfo(e) + else: + self.fail("Negative Test Failed - Exception DID NOT Occurred Under iso download ") + + @attr(tags=["advanced", "smoke"], required_hardware="true") + def test_04_create_iso_with_no_checksum(self): + self.iso.checksum = None + iso = self.registerIso(self.iso) + self.download(self.apiclient, iso.id) + + def registerIso(self, cmd): + iso = self.apiclient.registerIso(cmd)[0] + self.cleanup.append(iso) + return iso + + def getOsType(self, param): + cmd = listOsTypes.listOsTypesCmd() + cmd.description = param + return self.apiclient.listOsTypes(cmd)[0].id + + def download(self, apiclient, iso_id, retries=12, interval=5): + """Check if template download will finish in 1 minute""" + while retries > -1: + time.sleep(interval) + iso_response = Iso.list( + apiclient, + id=iso_id + ) + + if isinstance(iso_response, list): + iso = iso_response[0] + if not hasattr(iso, 'status') or not iso or not iso.status: + retries = retries - 1 + continue + + # If iso is ready, + # iso.status = Download Complete + # Downloading - x% Downloaded + # if Failed + # Error - Any other string + if 'Failed' in iso.status: + raise Exception( + "Failed to download iso: status - %s" % + iso.status) + + elif iso.status == 'Successfully Installed' and iso.isready: + return + + elif 'Downloaded' in iso.status: + retries = retries - 1 + continue + + elif 'Installing' not in iso.status: + if retries >= 0: + retries = retries - 1 + continue + raise Exception( + "Error in downloading iso: status - %s" % + iso.status) + + else: + retries = retries - 1 + raise Exception("Template download failed exception.") \ No newline at end of file diff --git a/test/integration/smoke/test_templates.py b/test/integration/smoke/test_templates.py index a621a0a6c12..8d76de36a59 100644 --- a/test/integration/smoke/test_templates.py +++ b/test/integration/smoke/test_templates.py @@ -17,6 +17,8 @@ """ BVT tests for Templates ISO """ #Import Local Modules +from marvin.cloudstackException import * +from marvin.cloudstackAPI import * from marvin.codes import FAILED from marvin.cloudstackTestCase import cloudstackTestCase, unittest from marvin.cloudstackAPI import listZones @@ -82,6 +84,203 @@ def create(apiclient, services, volumeid=None, account=None, domainid=None, proj cmd.projectid = projectid return apiclient.createTemplate(cmd) +class TestCreateTemplateWithChecksum(cloudstackTestCase): + def setUp(self): + self.testClient = super(TestCreateTemplateWithChecksum, self).getClsTestClient() + self.apiclient = self.testClient.getApiClient() + self.dbclient = self.testClient.getDbConnection() + self.cleanup = [] + + self.services = self.testClient.getParsedTestDataConfig() + self.unsupportedHypervisor = False + self.hypervisor = self.testClient.getHypervisorInfo() + if self.hypervisor.lower() in ['lxc']: + # Template creation from root volume is not supported in LXC + self.unsupportedHypervisor = True + return + + # Get Zone, Domain and templates + self.domain = get_domain(self.apiclient) + self.zone = get_zone(self.apiclient, self.testClient.getZoneForTests()) + + if "kvm" in self.hypervisor.lower(): + self.test_template = registerTemplate.registerTemplateCmd() + self.test_template = registerTemplate.registerTemplateCmd() + self.test_template.checksum = "{SHA-1}" + "bf580a13f791d86acf3449a7b457a91a14389264" + self.test_template.hypervisor = self.hypervisor + self.test_template.zoneid = self.zone.id + self.test_template.name = 'test sha-2333' + self.test_template.displaytext = 'test sha-1' + self.test_template.url = "http://dl.openvm.eu/cloudstack/macchinina/x86_64/macchinina-kvm.qcow2.bz2" + self.test_template.format = "QCOW2" + self.test_template.ostypeid = self.getOsType("Other Linux (64-bit)") + self.md5 = "ada77653dcf1e59495a9e1ac670ad95f" + self.sha256 = "0efc03633f2b8f5db08acbcc5dc1be9028572dfd8f1c6c8ea663f0ef94b458c5" + + if "vmware" in self.hypervisor.lower(): + self.test_template = registerTemplate.registerTemplateCmd() + self.test_template = registerTemplate.registerTemplateCmd() + self.test_template.checksum = "{SHA-1}" + "b25d404de8335b4348ff01e49a95b403c90df466" + self.test_template.hypervisor = self.hypervisor + self.test_template.zoneid = self.zone.id + self.test_template.name = 'test sha-2333' + self.test_template.displaytext = 'test sha-1' + self.test_template.url = "http://dl.openvm.eu/cloudstack/macchinina/x86_64/macchinina-vmware.ova" + self.test_template.format = "OVA" + self.test_template.ostypeid = self.getOsType("Other Linux (64-bit)") + self.md5 = "d6d97389b129c7d898710195510bf4fb" + self.sha256 = "f57b59f118ab59284a70d6c63229d1de8f2d69bffc5a82b773d6c47e769c12d9" + + if "xen" in self.hypervisor.lower(): + self.test_template = registerTemplate.registerTemplateCmd() + self.test_template = registerTemplate.registerTemplateCmd() + self.test_template.checksum = "{SHA-1}" + "427fad501d0d8a1d63b8600a9a469fbf91191314" + self.test_template.hypervisor = self.hypervisor + self.test_template.zoneid = self.zone.id + self.test_template.name = 'test sha-2333' + self.test_template.displaytext = 'test sha-1' + self.test_template.url = "http://dl.openvm.eu/cloudstack/macchinina/x86_64/macchinina-xen.vhd.bz2" + self.test_template.format = "VHD" + self.test_template.ostypeid = self.getOsType("Other Linux (64-bit)") + self.md5 = "54ebc933e6e07ae58c0dc97dfd37c824" + self.sha256 = "bddd9876021d33df9792b71ae4b776598680ac68ecf55e9d9af33c80904cc1f3" + + if self.unsupportedHypervisor: + self.skipTest("Skipping test because unsupported hypervisor\ + %s" % self.hypervisor) + return + + def tearDown(self): + try: + # Clean up the created templates + for temp in self.cleanup: + cmd = deleteTemplate.deleteTemplateCmd() + cmd.id = temp.id + cmd.zoneid = self.zone.id + self.apiclient.deleteTemplate(cmd) + + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) + return + + @attr(tags=["advanced", "smoke"], required_hardware="true") + def test_02_create_template_with_checksum_sha1(self): + template = self.registerTemplate(self.test_template) + self.download(self.apiclient, template.id) + + @attr(tags=["advanced", "smoke"], required_hardware="true") + def test_03_create_template_with_checksum_sha256(self): + self.test_template.checksum = "{SHA-256}" + self.sha256 + template = self.registerTemplate(self.test_template) + self.download(self.apiclient, template.id) + + @attr(tags=["advanced", "smoke"], required_hardware="true") + def test_04_create_template_with_checksum_md5(self): + self.test_template.checksum = "{md5}" + self.md5 + template = self.registerTemplate(self.test_template) + self.download(self.apiclient, template.id) + + @attr(tags=["advanced", "smoke"], required_hardware="true") + def test_02_1_create_template_with_checksum_sha1_negative(self): + self.test_template.checksum = "{sha-1}" + "someInvalidValue" + template = self.registerTemplate(self.test_template) + + try: + self.download(self.apiclient, template.id) + except Exception as e: + print "Negative Test Passed - Exception Occurred Under template download " \ + "%s" % GetDetailExceptionInfo(e) + else: + self.fail("Negative Test Failed - Exception DID NOT Occurred Under template download ") + + @attr(tags=["advanced", "smoke"], required_hardware="true") + def test_03_1_create_template_with_checksum_sha256_negative(self): + self.test_template.checksum = "{SHA-256}" + "someInvalidValue" + template = self.registerTemplate(self.test_template) + + try: + self.download(self.apiclient, template.id) + except Exception as e: + print "Negative Test Passed - Exception Occurred Under template download " \ + "%s" % GetDetailExceptionInfo(e) + else: + self.fail("Negative Test Failed - Exception DID NOT Occurred Under template download ") + + @attr(tags=["advanced", "smoke"], required_hardware="true") + def test_04_1_create_template_with_checksum_md5_negative(self): + self.test_template.checksum = "{md5}" + "someInvalidValue" + template = self.registerTemplate(self.test_template) + + try: + self.download(self.apiclient, template.id) + except Exception as e: + print "Negative Test Passed - Exception Occurred Under template download " \ + "%s" % GetDetailExceptionInfo(e) + else: + self.fail("Negative Test Failed - Exception DID NOT Occurred Under template download ") + + @attr(tags=["advanced", "smoke"], required_hardware="true") + def test_05_create_template_with_no_checksum(self): + self.test_template.checksum = None + template = self.registerTemplate(self.test_template) + self.download(self.apiclient, template.id) + + def registerTemplate(self, cmd): + temp = self.apiclient.registerTemplate(cmd)[0] + self.cleanup.append(temp) + return temp + + def getOsType(self, param): + cmd = listOsTypes.listOsTypesCmd() + cmd.description = param + return self.apiclient.listOsTypes(cmd)[0].id + + def download(self, apiclient, template_id, retries=12, interval=5): + """Check if template download will finish in 1 minute""" + while retries > -1: + time.sleep(interval) + template_response = Template.list( + apiclient, + id=template_id, + zoneid=self.zone.id, + templatefilter='self' + ) + + if isinstance(template_response, list): + template = template_response[0] + if not hasattr(template, 'status') or not template or not template.status: + retries = retries - 1 + continue + + # If template is ready, + # template.status = Download Complete + # Downloading - x% Downloaded + # if Failed + # Error - Any other string + if 'Failed' in template.status: + raise Exception( + "Failed to download template: status - %s" % + template.status) + + elif template.status == 'Download Complete' and template.isready: + return + + elif 'Downloaded' in template.status: + retries = retries - 1 + continue + + elif 'Installing' not in template.status: + if retries >= 0: + retries = retries - 1 + continue + raise Exception( + "Error in downloading template: status - %s" % + template.status) + + else: + retries = retries - 1 + raise Exception("Template download failed exception.") + class TestCreateTemplate(cloudstackTestCase): diff --git a/utils/src/main/java/org/apache/cloudstack/utils/security/ChecksumValue.java b/utils/src/main/java/org/apache/cloudstack/utils/security/ChecksumValue.java new file mode 100644 index 00000000000..e47bf393db2 --- /dev/null +++ b/utils/src/main/java/org/apache/cloudstack/utils/security/ChecksumValue.java @@ -0,0 +1,86 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.utils.security; + +import java.util.Objects; + +import org.apache.commons.lang.StringUtils; + +public class ChecksumValue { + String checksum; + String algorithm = "MD5"; + public ChecksumValue(String algorithm, String checksum) { + this.algorithm = algorithm; + this.checksum = checksum; + } + public ChecksumValue(String digest) { + digest = StringUtils.strip(digest); + this.algorithm = algorithmFromDigest(digest); + this.checksum = stripAlgorithmFromDigest(digest); + } + + @Override + public String toString() { + return '{' + algorithm + '}'+ checksum; + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + ChecksumValue that = (ChecksumValue)o; + return Objects.equals(getChecksum(), that.getChecksum()) && Objects.equals(getAlgorithm(), that.getAlgorithm()); + } + + @Override + public int hashCode() { + return Objects.hash(getChecksum(), getAlgorithm()); + } + + public String getChecksum() { + return checksum; + } + + public String getAlgorithm() { + return algorithm; + } + + private static String stripAlgorithmFromDigest(String digest) { + if(StringUtils.isNotEmpty(digest)) { + int s = digest.indexOf('{');// only assume a + int e = digest.indexOf('}'); + if (s == 0 && e > s) { // we have an algorithm name of at least 1 char + return digest.substring(e+1); + } + } + // we assume digest is alright if there is no algorithm at the start + return digest; + } + + private static String algorithmFromDigest(String digest) { + if(StringUtils.isNotEmpty(digest)) { + int s = digest.indexOf('{'); + int e = digest.indexOf('}'); + if (s == 0 && e > s+1) { // we have an algorithm name of at least 1 char + return digest.substring(s+1,e); + } // else if no algoritm + } // or if no digest at all + return "MD5"; + } +} diff --git a/utils/src/main/java/org/apache/cloudstack/utils/security/DigestHelper.java b/utils/src/main/java/org/apache/cloudstack/utils/security/DigestHelper.java new file mode 100644 index 00000000000..67adf74189d --- /dev/null +++ b/utils/src/main/java/org/apache/cloudstack/utils/security/DigestHelper.java @@ -0,0 +1,96 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.utils.security; + +import java.io.IOException; +import java.io.InputStream; +import java.math.BigInteger; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.HashMap; +import java.util.Map; + +public class DigestHelper { + + public static ChecksumValue digest(String algorithm, InputStream is) throws NoSuchAlgorithmException, IOException { + MessageDigest digest = MessageDigest.getInstance(algorithm); + ChecksumValue checksum = null; + byte[] buffer = new byte[8192]; + int read = 0; + while ((read = is.read(buffer)) > 0) { + digest.update(buffer, 0, read); + } + byte[] md5sum = digest.digest(); + // TODO make sure this is valid for all types of checksums !?! + BigInteger bigInt = new BigInteger(1, md5sum); + checksum = new ChecksumValue(digest.getAlgorithm(), getPaddedDigestString(digest,bigInt)); + return checksum; + } + + public static boolean check(String checksum, InputStream is) throws IOException, NoSuchAlgorithmException { + ChecksumValue toCheckAgainst = new ChecksumValue(checksum); + String algorithm = toCheckAgainst.getAlgorithm(); + ChecksumValue result = digest(algorithm,is); + return result.equals(toCheckAgainst); + } + + public static String getPaddedDigest(String algorithm, String inputString) throws NoSuchAlgorithmException { + MessageDigest digest = MessageDigest.getInstance(algorithm); + String checksum; + digest.reset(); + BigInteger pwInt = new BigInteger(1, digest.digest(inputString.getBytes())); + return getPaddedDigestString(digest, pwInt); + } + + private static String getPaddedDigestString(MessageDigest digest, BigInteger pwInt) { + String checksum; + String pwStr = pwInt.toString(16); + // we have half byte string representation, so + int padding = 2*digest.getDigestLength() - pwStr.length(); + StringBuffer sb = new StringBuffer(); + for (int i = 0; i < padding; i++) { + sb.append('0'); // make sure the MD5 password is 32 digits long + } + sb.append(pwStr); + checksum = sb.toString(); + return checksum; + } + + static final Map paddingLengths = creatPaddingLengths(); + + private static final Map creatPaddingLengths() { + Map map = new HashMap<>(); + map.put("MD5", 32); + map.put("SHA-1", 40); + map.put("SHA-224", 56); + map.put("SHA-256", 64); + map.put("SHA-384", 96); + map.put("SHA-512", 128); + return map; + } + + public static boolean isAlgorithmSupported(String checksum) { + ChecksumValue toCheckAgainst = new ChecksumValue(checksum); + String algorithm = toCheckAgainst.getAlgorithm(); + try { + MessageDigest.getInstance(algorithm); + } catch (NoSuchAlgorithmException e) { + return false; + } + return true; + } +} diff --git a/utils/src/test/java/org/apache/cloudstack/utils/security/DigestHelperTest.java b/utils/src/test/java/org/apache/cloudstack/utils/security/DigestHelperTest.java new file mode 100644 index 00000000000..45408827376 --- /dev/null +++ b/utils/src/test/java/org/apache/cloudstack/utils/security/DigestHelperTest.java @@ -0,0 +1,102 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.utils.security; + +import java.io.IOException; +import java.io.InputStream; +import java.io.UnsupportedEncodingException; + +import com.amazonaws.util.StringInputStream; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +public class DigestHelperTest { + + private final static String INPUT_STRING = "01234567890123456789012345678901234567890123456789012345678901234567890123456789\n"; + private final static String INPUT_STRING_NO2 = "01234567890123456789012345678901234567890123456789012345678901234567890123456789b\n"; + private final static String INPUT_STRING_NO3 = "01234567890123456789012345678901234567890123456789012345678901234567890123456789h\n"; + private final static String SHA256_CHECKSUM = "{SHA-256}c6ab15af7842d23d3c06c138b53a7d09c5e351a79c4eb3c8ca8d65e5ce8900ab"; + private final static String SHA1_CHECKSUM = "{SHA-1}49e4b2f4292b63e88597c127d11bc2cc0f2ca0ff"; + private final static String MD5_CHECKSUM = "{MD5}d141a8eeaf6bba779d1d1dc5102a81c5"; + private final static String ZERO_PADDED_MD5_CHECKSUM = "{MD5}0e51dfa74b87f19dd5e0124d6a2195e3"; + private final static String ZERO_PADDED_SHA256_CHECKSUM = "{SHA-256}08b5ae0c7d7d45d8ed406d7c3c7da695b81187903694314d97f8a37752a6b241"; + private static final String MD5 = "MD5"; + private static final String SHA_256 = "SHA-256"; + private static InputStream inputStream; + private InputStream inputStream2; + + + @Test + public void check_SHA256() throws Exception { + Assert.assertTrue(DigestHelper.check(SHA256_CHECKSUM, inputStream)); + } + + @Test + public void check_SHA1() throws Exception { + Assert.assertTrue(DigestHelper.check(SHA1_CHECKSUM, inputStream)); + } + + @Test + public void check_MD5() throws Exception { + Assert.assertTrue(DigestHelper.check(MD5_CHECKSUM, inputStream)); + } + + @Test + public void testDigestSHA256() throws Exception { + String result = DigestHelper.digest(SHA_256, inputStream).toString(); + Assert.assertEquals(SHA256_CHECKSUM, result); + } + + @Test + public void testDigestSHA1() throws Exception { + String result = DigestHelper.digest("SHA-1", inputStream).toString(); + Assert.assertEquals(SHA1_CHECKSUM, result); + } + + @Test + public void testDigestMD5() throws Exception { + String result = DigestHelper.digest(MD5, inputStream).toString(); + Assert.assertEquals(MD5_CHECKSUM, result); + } + + @Test + public void testZeroPaddedDigestMD5() throws Exception { + inputStream2 = new StringInputStream(INPUT_STRING_NO2); + String result = DigestHelper.digest(MD5, inputStream2).toString(); + Assert.assertEquals(ZERO_PADDED_MD5_CHECKSUM, result); + } + + @Test + public void testZeroPaddedDigestSHA256() throws Exception { + inputStream2 = new StringInputStream(INPUT_STRING_NO3); + String result = DigestHelper.digest(SHA_256, inputStream2).toString(); + Assert.assertEquals(ZERO_PADDED_SHA256_CHECKSUM, result); + } + + @BeforeClass + public static void init() throws UnsupportedEncodingException { + inputStream = new StringInputStream(INPUT_STRING); + } + @Before + public void reset() throws IOException { + inputStream.reset(); + } +} + +//Generated with love by TestMe :) Please report issues and submit feature requests at: http://weirddev.com/forum#!/testme \ No newline at end of file