如何使用nodejs将过滤后的数据从dynamodb导出到csv？

Question

I want to export filtered data from dynamodb to csv. I have a function for filtering data using name and id..now to i need to export the filtered data to excel..how can i fix this? I had seen this npm dynamodb-csv . but this didnt solve my problem..

Answer 1

I'm using the following code snippet for export and import:

const AWS = require('aws-sdk');
const dynamoDb = new AWS.DynamoDB.DocumentClient({ region: 'eu-central-1' });
const READ_FROM_TABLE = "example-table-prod"
const STORE_TO_TABLE = "example-table-dev"
const fs = require('fs');

async function loadAllResults() {
  var results = []
  var currentResults = await loadAdditionalResults()
  results = results.concat(currentResults.Items)
  while (currentResults.LastEvaluatedKey) {
    currentResults = await loadAdditionalResults(currentResults.LastEvaluatedKey)
    results = results.concat(currentResults.Items)
  }
  fs.writeFile("output.json", JSON.stringify(results, null, 2), () => {
    console.log(results.length)
  })
  return results
}

async function loadAdditionalResults(start) {
  console.log("Loading more results...")
  var params = {
    TableName: READ_FROM_TABLE,
  }

  if (start) {
    params.ExclusiveStartKey = start
  }

  return new Promise((resolve, reject) => {
    dynamoDb.scan(params, (error, result) => {
      if (error) {
        console.log(error);
        reject(error)
      } else if (result) {
        resolve(result)
      } else {
        reject("Unknown error")
      }
    })
  })
}

async function storeAllResults(results) {
  var count = 0
  for (result of results) {
    await storeItem(result)
    count++
    console.log("Stored " + count + " / " + results.length)
  }
}

async function storeItem(item) {
  const params = {
    TableName: STORE_TO_TABLE,
    Item: item
  };

  return new Promise((resolve, reject) => {
    dynamoDb.put(params, function (error, data) {
      if (error) {
        console.log(error);
        reject({ error: 'Could not create item' });
      }
      resolve("Stored");
    });
  });
};

async function main() {
  let results = await loadAllResults()
  storeAllResults(results)
}


main()

Feel free to use whatever snippet of that is useful to you. Also note that you can convert the json to csv using https://www.npmjs.com/package/json2csv

Answer 2

I had the same issue and I used the above code to fix my problem. But I wanted to generate a CSV file from the DynamoDB table. Thanks, @Carl Ambroselli, I modify your code for my desired result.

I used "papaparse" in the above code and got my DynamoDB records into CSV format.

dynamodb-csv package has mentioned limitation as CSV column headers are determined based on the first row in your DynamoDB table.

We can pass column names in fields to overcome limitations.

const fs = require('fs');
const async = require('async');
const AWS = require('aws-sdk');
const Papa = require("papaparse");


const DYNAMODB_REGION = 'XXXXXXXX';

AWS.config.update({
  accessKeyId: 'XXXXXXXXXXX',
  secretAccessKey: 'XXXXXXXXXX',
  region: 'XXXXXX',
});

const docClient = new AWS.DynamoDB.DocumentClient({
  region: DYNAMODB_REGION
});


const READ_FROM_TABLE = "tableName"

main();

async function loadAllResults() {
  var results = [];
  var currentResults = await loadAdditionalResults();
  results = results.concat(currentResults.Items);

  while (currentResults.LastEvaluatedKey) {
    currentResults = await loadAdditionalResults(currentResults.LastEvaluatedKey)
    results = results.concat(currentResults.Items)
  }
  return results;
}

async function loadAdditionalResults(start) {
  console.log("Loading more results..."+READ_FROM_TABLE)
  var params = {
    TableName: READ_FROM_TABLE,
    //Limit: 1000,
  }

  if (start) {
    params.ExclusiveStartKey = start
  }

  return new Promise((resolve, reject) => {
    docClient.scan(params, (error, result) => {
      if (error) {
        console.log(error);
        reject(error);
      } else if (result) {
        resolve(result);
      } else {
        reject("Unknown error");
      }
    })
  })
}


async function main() {
  let tableData = await loadAllResults();
  var csv = Papa.unparse(tableData);
  fs.createWriteStream('test.csv', { flags: 'w' }).write(csv);

  /*var csv = Papa.unparse({
    fields: ["id", "productName", "price", "quantity"],
    data: tableData
    });
  fs.createWriteStream('test.csv', { flags: 'w' }).write(csv);*/
}

Output as