How to access S3 bucket through JavaScript?

Question

I've looked at some other posts and tried to replicate what they've done, but none of them seem to be running into the same issue I am. Basically, I'm trying to store the list of keys from a S3 bucket so I can iterate through them in my Vue app. I've got the code below, and you can see I have 3 console.log statements where I'm trying to print the value of files . The first one prints exactly what I expect it to, while the 2nd one prints [] , and the 3rd doesn't print at all. So for some reason it's not persisting the value of files outside of the s3.listObjectsV2() function, which means I can't access the actual files themselves in the app.

let AWS = require("aws-sdk");
AWS.config.update({
  accessKeyId: process.env.VUE_APP_ACCESS_KEY,
  secretAccessKey: process.env.VUE_APP_ACCESS_KEY,
  region: "us-east-1",
});
let s3 = new AWS.S3();

let params = {
  Bucket: "my-bucket",
  Delimiter: "",
};

let getS3Files = () => {
  let files = [];
  s3.listObjectsV2(params, function (err, data) {
    if (data) {
      data.Contents.forEach((file) => {
        files.push({
          fileName: file.Key,
          fileDate: file.LastModified,
        });
      });
      console.log(files);
    }
  });
  console.log(files);
  if (files.length > 0) {
    files = files.sort((a, b) => b.fileDate - a.fileDate);
    console.log(files);
  }
  return files;
};

getS3Files();

Answer 1

It is because you are not waiting until the s3.listObjectsV2 completes fetching its data. Although s3.listObjectsV2 does not seem like a function you should wait until it completes, it needs time to send a request to s3 and get the data you requested.

In the code you provided, the second console.log will most probably execute before the first console.log and by that time you would not have any data pushed to files . Hence it will print [] . And before the third log, it is checking whether you have any element in your files which would again equal to false since files = [] .

So, you need to wrap this around a Promise and wait until it completes.

let getS3Files = async () => {
  let files = [];

  await new Promise(function (resolve, reject) {
    s3.listObjectsV2(params, function (err, data) {
      if (data) {
        data.Contents.forEach((file) => {
          files.push({
            fileName: file.Key,
            fileDate: file.LastModified,
          });
        });
        console.log(files);
        resolve();
      } else {
        reject(err);
      }
    });
  });
  console.log(files);
  if (files.length > 0) {
    files = files.sort((a, b) => b.fileDate - a.fileDate);
    console.log(files);
  }
  return files;
};

await getS3Files();

Answer 2

I am not familiar with the s3 api, but I understand listObjectsV2 is an asynchronous operation, in consequence files will be empty because it happens outside the asynchronous call and is executed before having values

Please try

let AWS = require("aws-sdk");

AWS.config.update({
  accessKeyId: process.env.VUE_APP_ACCESS_KEY,
  secretAccessKey: process.env.VUE_APP_ACCESS_KEY,
  region: "us-east-1",
});

let s3 = new AWS.S3();
let params = {
  Bucket: "my-bucket",
  Delimiter: "",
};

let getS3Files = (callback) => {
  s3.listObjectsV2(params, function (err, data) {
    callback(data);
  });
};

getS3Files((data) => {
  const files = data.Contents.map((file) => ({
    fileName: file.key,
    fileDate: file.LastModified,
  })).sort((a, b) => b.fileDate - a.fileDate);

  return files;
});

In this case, the getS3Files receives a callback that will contain the data, which you will process in the function call.

Take a look to this question How do I return the response from an asynchronous call?