etan39
/
cloudstack
mirror of https://github.com/apache/cloudstack.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

listDirectory method updated to use ObjectListing.isTruncated(). 

Because buckets can contain a virtually unlimited number of keys, the
complete results of a list query can be extremely large. To manage large
result sets, Amazon S3 uses pagination to split them into multiple
responses.

Signed-off-by: Rajani Karuturi <rajanikaruturi@gmail.com>

This closes #25

(cherry picked from commit 0b6c540a20)
Signed-off-by: Rohit Yadav <rohit.yadav@shapeblue.com>
This commit is contained in:
santhosh 2014-10-24 21:15:29 +05:30 committed by Rohit Yadav
parent 8069b17250
commit 0ae5912f00
1 changed files with 14 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
utils/src/com/cloud/utils/S3Utils.java
View File

@ -57,13 +57,14 @@ import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.model.Bucket;
import com.amazonaws.services.s3.model.CannedAccessControlList;
import com.amazonaws.services.s3.model.GetObjectRequest;
import com.amazonaws.services.s3.model.ListObjectsRequest;
import com.amazonaws.services.s3.model.ObjectListing;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.model.S3Object;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.amazonaws.services.s3.transfer.TransferManager;
import com.amazonaws.services.s3.transfer.Upload;
import com.cloud.utils.exception.CloudRuntimeException;
public final class S3Utils {
@ -338,9 +339,19 @@ public final class S3Utils {
private static List<S3ObjectSummary> listDirectory(final String bucketName, final String directory, final AmazonS3 client) {
final List<S3ObjectSummary> objects = client.listObjects(bucketName, directory + SEPARATOR).getObjectSummaries();
List<S3ObjectSummary> objects = new ArrayList<S3ObjectSummary>();
ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName).withPrefix(directory + SEPARATOR);
ObjectListing objectListing;
if (objects == null) {
do {
objectListing = client.listObjects(listObjectsRequest);
if (objectListing != null )
objects.addAll(objectListing.getObjectSummaries());
listObjectsRequest.setMarker(objectListing.getNextMarker());
} while (objectListing.isTruncated());
if (objects.isEmpty()) {
return emptyList();
}