如何使用 node.js 從`AWS dynamodb` 獲取/掃描所有項目

Question

如何使用node.js從AWS dynamodb獲取/掃描所有項目。 我在這里發布我的代碼。

var docClient = new aws.DynamoDB.DocumentClient();
    var params = {
    TableName:"users",
    KeyConditionExpression:"user_status=:status",
    ExpressionAttributeValues: {
        ":status": "Y"
    }
    };

    var queryExecute = function(callback) {
        docClient.query(params,function(err,result) {
            if(err) {
                console.log(err)
                callback(err);
                } else {
                console.log(result);

                if(result.LastEvaluatedKey) {
                    params.ExclusiveStartKey = result.LastEvaluatedKey;
                    queryExecute(callback);
                    } else {
                        callback(err,items);
                    }
                }
            });
        }
        queryExecute(callback); 

這給了我以下錯誤。

ValidationException: Query condition missed key schema element: `user_id`.

這里的主鍵是 user_id。 我不想在我的查詢條件中使用它，因為如果我在KeyConditionExpression提到了主鍵，我需要設置一個值。 可能是我錯了。 但是，請建議我從dynamodb獲取所有項目的好方法，它具有user_status = "Y"

Answer 1

這對我有用：

export const scanTable = async (tableName) => {
    const params = {
        TableName: tableName,
    };

    const scanResults = [];
    const items;
    do{
        items =  await documentClient.scan(params).promise();
        items.Items.forEach((item) => scanResults.push(item));
        params.ExclusiveStartKey  = items.LastEvaluatedKey;
    }while(typeof items.LastEvaluatedKey !== "undefined");
    
    return scanResults;

};

Answer 2

如果您想在不使用 Hash 鍵值的情況下從 DynamoDB 獲取數據，則需要使用Scan API 。

注意： Scan API 讀取表中的所有項目以獲取結果。 因此，在 DynamoDB 中這是一項代價高昂的操作。

替代方法：使用 GSI

上述場景的掃描碼：-

var docClient = new AWS.DynamoDB.DocumentClient();

var params = {
    TableName: "users",
    FilterExpression: "#user_status = :user_status_val",
    ExpressionAttributeNames: {
        "#user_status": "user_status",
    },
    ExpressionAttributeValues: { ":user_status_val": 'somestatus' }

};

docClient.scan(params, onScan);
var count = 0;

function onScan(err, data) {
    if (err) {
        console.error("Unable to scan the table. Error JSON:", JSON.stringify(err, null, 2));
    } else {        
        console.log("Scan succeeded.");
        data.Items.forEach(function(itemdata) {
           console.log("Item :", ++count,JSON.stringify(itemdata));
        });

        // continue scanning if we have more items
        if (typeof data.LastEvaluatedKey != "undefined") {
            console.log("Scanning for more...");
            params.ExclusiveStartKey = data.LastEvaluatedKey;
            docClient.scan(params, onScan);
        }
    }
}

Answer 3

AWS 文檔示例對我不起作用。 @Hank 方法成功了。

在 lambda 中使用處理程序：

const AWS = require('aws-sdk');
const docClient = new AWS.DynamoDB.DocumentClient({
    // optional tuning - 50% faster(cold) / 20% faster(hot)
    apiVersion: '2012-08-10',
    sslEnabled: false,
    paramValidation: false,
    convertResponseTypes: false
});

const tableName = 'series';

exports.handler = async (event, context, callback) => {
    let params = { TableName: tableName };

    let scanResults = [];
    let items;

    do {
        items = await docClient.scan(params).promise();
        items.Items.forEach((item) => scanResults.push(item));
        params.ExclusiveStartKey = items.LastEvaluatedKey;
    } while (typeof items.LastEvaluatedKey != "undefined");

    callback(null, scanResults);
};

Answer 4

使用承諾和異步

const aws = require('aws-sdk');
aws.config.update({ region: 'us-east-1' });
const documentClient = new aws.DynamoDB.DocumentClient();

const scanAll = async (params) => {
  let lastEvaluatedKey = 'dummy'; // string must not be empty
  const itemsAll = [];
  while (lastEvaluatedKey) {
    const data = await documentClient.scan(params).promise();
    itemsAll.push(...data.Items);
    lastEvaluatedKey = data.LastEvaluatedKey;
    if (lastEvaluatedKey) {
      params.ExclusiveStartKey = lastEvaluatedKey;
    }
  }
  return itemsAll;
}

像這樣使用

const itemsAll = scanAll(params);

查詢代碼相同（只需將掃描替換為查詢）

Answer 5

const AWS = require('aws-sdk');
const docClient = new AWS.DynamoDB.DocumentClient({
    // optional tuning - 50% faster(cold) / 20% faster(hot)
    apiVersion: '2012-08-10',
    sslEnabled: false,
    paramValidation: false,
    convertResponseTypes: false,
    region: 'us-east-2' // put your region
});
const tableName = 'tableName'; // put your tablename

exports.handler = async (event, context, callback) => {
    let params = { TableName: tableName };

    let scanResults = [];
    let items;

    do {
        items = await docClient.scan(params).promise();
        items.Items.forEach((item) => scanResults.push(item));
        params.ExclusiveStartKey = items.LastEvaluatedKey;
    } while (typeof items.LastEvaluatedKey != "undefined");

    callback(null, scanResults);
};

Answer 6

我使用這樣的承諾：

let AWS = require('aws-sdk');
let docClient = new AWS.DynamoDB.DocumentClient();

async function dbRead(params) {
    let promise = docClient.scan(params).promise();
    let result = await promise;
    let data = result.Items;
    if (result.LastEvaluatedKey) {
        params.ExclusiveStartKey = result.LastEvaluatedKey;
        data = data.concat(await dbRead(params));
    }
    return data;
}

並使用它：

let params = {
  TableName: 'Table'
};
let data = await dbRead(params);

Answer 7

一個以JSON格式返回數據的節點快遞解決方案：

let datapack=[];
item = {
        TableName: ddbTable,
        FilterExpression: "aws = :e AND begins_with ( Id, :t )",
        ExpressionAttributeValues: {
            ":t"    :   "contact",
            ":e"    :   aws
        },
        ProjectionExpression: "Id,FirstName,LastName,cEmail",
    };
    docClient.scan(item, onScan);
    function onScan(err, data) {
        if (err) {
            console.error("Unable to scan the table. Error JSON:", JSON.stringify(err, null, 2));
        } else {        
            datapack = datapack.concat(data.Items);
            });
            if (typeof data.LastEvaluatedKey != "undefined") {
                item.ExclusiveStartKey = data.LastEvaluatedKey;
                docClient.scan(item, onScan);
            } else {
                res.json(datapack);
            }
        }
    }

Answer 8

這是一個答案，它將提供索引結果，而不是使用昂貴的掃描，也是 JSON 格式的節點/快遞。 注意 docClient.query 的使用：

 datapack=[];
    item = {
        TableName: ddbTable,
        IndexName: "cEmailIndex",
        KeyConditionExpression : "aws = :e AND begins_with ( cEmail, :t )",
        ExpressionAttributeValues: {
            ":t"    :   search,
            ":e"    :   aws
        },
        ProjectionExpression: "Id,FirstName,LastName,cEmail",
    };
    docClient.query(item, onScan);
    function onScan(err, data) {
        if (err) {
            console.error("Unable to scan the table. Error JSON:", JSON.stringify(err, null, 2));
        } else {
            datapack = datapack.concat(data.Items);
            if (typeof data.LastEvaluatedKey != "undefined") {
                item.ExclusiveStartKey = data.LastEvaluatedKey;
                docClient.query(item, onScan);
            } else {
                // console.log(JSON.stringify(datapack));
                res.json(datapack);
            }
        }
    }

Answer 9

您可以使用@aws/dynamodb-query-iterator 中的 ScanPaginator ：

import { ScanPaginator } from '@aws/dynamodb-query-iterator';
import DynamoDB = require('aws-sdk/clients/dynamodb');

const paginator = new ScanPaginator(
  new DynamoDB.DocumentClient(),
  {
    TableName: "users",
    FilterExpression: "#user_status = :user_status_val",
    ExpressionAttributeNames: {
      "#user_status": "user_status",
    },
    ExpressionAttributeValues: { ":user_status_val": 'somestatus' }
  }
);

for await (const page of paginator) {
    // do something with `page`, e.g. myFunction(page.Items)
}

Answer 10

對於那些不使用 AWS.DynamoDB.DocumentClient 的人，此解決方案將起作用。 為了便於閱讀和使用 async/await，我已將功能拆分為多個模塊。

const AWS = require("aws-sdk");
AWS.config.update({
    // update table region here
    region: "us-west-2"
});
var dynamodb = new AWS.DynamoDB();
const performAsynScanOperation = (scanParams) => {
    return new Promise((resolve, reject) => {
        dynamodb.scan(scanParams, function (err, responseData) {
            if (err) {
                reject(err)
            } else {
                resolve(responseData)
            }
        })
    })
}

const getAllRecords = async (tableName) => {
    let allItems = [];
    let LastEvaluatedKeyFlag = true;
    let scanParams = { TableName: tableName }
    while (LastEvaluatedKeyFlag) {
        let responseData = await performAsynScanOperation(scanParams)
        let batchItems = responseData.Items;
        allItems = allItems.concat(batchItems);
        if (responseData.LastEvaluatedKey) {
            LastEvaluatedKeyFlag = true;
            console.log('LastEvaluatedKey', responseData.LastEvaluatedKey)
            scanParams.ExclusiveStartKey = responseData.LastEvaluatedKey
        } else {
            LastEvaluatedKeyFlag = false;
        }
    }
    return allItems;
}
getAllRecords('<Name of table>').then((allItems)=>{
  console.log(allItems)
})

Answer 11

@大衛，

我試圖使您的示例工作（JSON）

// Setting up our dependencies
const AWS = require('aws-sdk');
const https = require('https');
const dynamodb = new AWS.DynamoDB.DocumentClient();

// Creating an eviornment variable to store DynamoDB table name
const TABLE_NAME = process.env.table_name;

exports.handler = function(event, context, callback)  {
    var siteNum = "7261"; // event.siteNbr;
    var res;
    let datapack =[];
    let item = {
            TableName:TABLE_NAME,
            IndexName: "SiteNumIndex",
            KeyConditionExpression: "SiteNum = :siteNum",
            "ExpressionAttributeValues": {
                ":siteNum": siteNum
            }
        };

        console.log("siteNbr: " + siteNum)   ;

    dynamodb.query(item, onScan);

    function onScan(err, data) {
        if (err) {
            console.error("Unable to scan the table. Error JSON:", JSON.stringify(err, null, 2));
        } else {
            datapack = datapack.concat(data.Items);
            if (typeof data.LastEvaluatedKey != "undefined") {
                item, onScan.ExclusiveStartKey = data.LastEvaluatedKey;
                dynamodb.query(item, onScan);
            } else {
                console.log(JSON.stringify(datapack));
                //data.json(datapack);
            }
        }
    }
};

最后一個console.log確實編寫了JSON，但是響應返回null。 請原諒我的無知，我還不是node.js開發人員。 在您的示例中，什么是res？ 它從何而來？ 我習慣於C＃，在其中我會做類似“返回JSON.stringify（datapack）;”的操作

Answer 12

這是掃描所有記錄的替代品：

const scanAll = async (params) => {
    let all = [];
    while (true) {
        let data = await new Promise((resolve, reject) => {
            db.scan(params, function (err, data) {
                if (err)
                    reject(err);
                else
                    resolve(data);
            });
        });
        all = all.concat(data.Items);
        if (data.LastEvaluatedKey)
            params.ExclusiveStartKey = data.LastEvaluatedKey;
        else
            break;
    }
    return all;
};

用法：

scanAll(query)
    .catch((err) => {

    })
    .then((records) => {

    });
}

Answer 13

scan 方法讀取表中的每一項並返回表中的所有數據。 您可以提供一個可選的 filter_expression，以便只返回與您的條件匹配的項目。 但是，只有在掃描了整個表之后才應用過濾器。 參考

我正在分享重構的onScan函數希望它onScan幫助。

var AWS = require("aws-sdk");

AWS.config.update({
    region: "us-west-2",
    endpoint: "http://localhost:8000"
});

var docClient = new AWS.DynamoDB.DocumentClient();

async function read() {
        const params = {
            TableName: "tableName"
            // options can be passed here e.g.
            // FilterExpression: "#yr between :start_yr and :end_yr",
        };

        let items = [];
        return new Promise((resolve, reject) => {
            function onScan(err, data) {
                if (err) {
                    console.error("Unable to scan the table. Error JSON:", JSON.stringify(err, null, 2));
                    reject();
                } else {
                    items = items.concat(data.Items);

                    // continue scanning if we have more items, because
                    // scan can retrieve a maximum of 1MB of data
                    if (typeof data.LastEvaluatedKey !== "undefined") {
                        params.ExclusiveStartKey = data.LastEvaluatedKey;
                        docClient.scan(params, onScan);
                    } else {
                        resolve(items);
                    }
                }
            }
            docClient.scan(params, onScan);
        });
    }