在 Promise 中获取存储桶中的所有 Amazon S3 文件

Question

I'm trying to grab thousands of files from Amazon S3 within a Promise but I can't seem to figure out how to include the ContinuationToken within if the list is truncated and gather it all together within the promise. I'm a novice with JS and could use some help. Here's what I have, so far:

getFiles()
    .then(filterFiles)
    .then(mapUrls)
;

function getFiles(token) {
    var params = {
        Bucket: bucket,
        MaxKeys: 5000,
        ContinuationToken: token
    };
    var allKeys = [];

    var p = new Promise(function(resolve, reject){
    s3.listObjectsV2(params, function(err, data) {
      if (err) { 
        return reject(err);
      }
      allKeys.push(data.Contents)
      if (data.IsTruncated) {
        s3.listObjectsV2({Bucket: bucket, MaxKeys: 5000, ContinuationToken: data.NextContinuationToken})
        console.log('Getting more images...');
        allKeys.push(data.Contents)
      }
      resolve(data.Contents);
    });
  });

  return p;
}

I need the function to continue to run until I've created a list of all objects in the bucket to return.

Answer 1

You need ContinuationToken the second time only.

var params = {
    Bucket: bucket,
    MaxKeys: 5000,
};

if (data.IsTruncated) {
    s3.listObjectsV2({...params, ContinuationToken: data.NextContinuationToken})

IMO, this is just a s3 function called twice, more like a nested call. Recursion is when a function keeps calling itself until a specified condition is met.

Read more about recursion: https://medium.com/@vickdayaram/recursion-caad288bf621

Answer 2

I was able to list all objects in the bucket using async/await and the code below to populate an array.

async function getFiles(objects = []) {
    const response = await s3.listObjectsV2(params).promise();
    response.Contents.forEach(obj => filelist.push(obj.Key));
    if (response.NextContinuationToken) {
        params.ContinuationToken = response.NextContinuationToken;
        await getFiles(params, objects);
    }
    console.log(filelist.length)
    return filelist;
}

Thanks to all who helped!