From 0ae5912f0019b9ea4c496ea138ecb57fc840ef23 Mon Sep 17 00:00:00 2001 From: santhosh Date: Fri, 24 Oct 2014 21:15:29 +0530 Subject: [PATCH] listDirectory method updated to use ObjectListing.isTruncated(). Because buckets can contain a virtually unlimited number of keys, the complete results of a list query can be extremely large. To manage large result sets, Amazon S3 uses pagination to split them into multiple responses. Signed-off-by: Rajani Karuturi This closes #25 (cherry picked from commit 0b6c540a203453580bb76a0d0a60be292fccfa0e) Signed-off-by: Rohit Yadav --- utils/src/com/cloud/utils/S3Utils.java | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/utils/src/com/cloud/utils/S3Utils.java b/utils/src/com/cloud/utils/S3Utils.java index d9b850d17dc..37aeb9138d0 100644 --- a/utils/src/com/cloud/utils/S3Utils.java +++ b/utils/src/com/cloud/utils/S3Utils.java @@ -57,13 +57,14 @@ import com.amazonaws.services.s3.AmazonS3Client; import com.amazonaws.services.s3.model.Bucket; import com.amazonaws.services.s3.model.CannedAccessControlList; import com.amazonaws.services.s3.model.GetObjectRequest; +import com.amazonaws.services.s3.model.ListObjectsRequest; +import com.amazonaws.services.s3.model.ObjectListing; import com.amazonaws.services.s3.model.ObjectMetadata; import com.amazonaws.services.s3.model.PutObjectRequest; import com.amazonaws.services.s3.model.S3Object; import com.amazonaws.services.s3.model.S3ObjectSummary; import com.amazonaws.services.s3.transfer.TransferManager; import com.amazonaws.services.s3.transfer.Upload; - import com.cloud.utils.exception.CloudRuntimeException; public final class S3Utils { @@ -338,9 +339,19 @@ public final class S3Utils { private static List listDirectory(final String bucketName, final String directory, final AmazonS3 client) { - final List objects = client.listObjects(bucketName, directory + SEPARATOR).getObjectSummaries(); + List objects = new ArrayList(); + ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName).withPrefix(directory + SEPARATOR); + ObjectListing objectListing; - if (objects == null) { + do { + objectListing = client.listObjects(listObjectsRequest); + + if (objectListing != null ) + objects.addAll(objectListing.getObjectSummaries()); + listObjectsRequest.setMarker(objectListing.getNextMarker()); + } while (objectListing.isTruncated()); + + if (objects.isEmpty()) { return emptyList(); }