[英]How to fetch/scan all items from `AWS dynamodb` using node.js
如何使用node.js
从AWS dynamodb
获取/扫描所有项目。 我在这里发布我的代码。
var docClient = new aws.DynamoDB.DocumentClient();
var params = {
TableName:"users",
KeyConditionExpression:"user_status=:status",
ExpressionAttributeValues: {
":status": "Y"
}
};
var queryExecute = function(callback) {
docClient.query(params,function(err,result) {
if(err) {
console.log(err)
callback(err);
} else {
console.log(result);
if(result.LastEvaluatedKey) {
params.ExclusiveStartKey = result.LastEvaluatedKey;
queryExecute(callback);
} else {
callback(err,items);
}
}
});
}
queryExecute(callback);
这给了我以下错误。
ValidationException: Query condition missed key schema element: `user_id`.
这里的主键是 user_id。 我不想在我的查询条件中使用它,因为如果我在KeyConditionExpression
提到了主键,我需要设置一个值。 可能是我错了。 但是,请建议我从dynamodb
获取所有项目的好方法,它具有user_status = "Y"
这对我有用:
export const scanTable = async (tableName) => {
const params = {
TableName: tableName,
};
const scanResults = [];
const items;
do{
items = await documentClient.scan(params).promise();
items.Items.forEach((item) => scanResults.push(item));
params.ExclusiveStartKey = items.LastEvaluatedKey;
}while(typeof items.LastEvaluatedKey !== "undefined");
return scanResults;
};
如果您想在不使用 Hash 键值的情况下从 DynamoDB 获取数据,则需要使用Scan API
。
注意: Scan API 读取表中的所有项目以获取结果。 因此,在 DynamoDB 中这是一项代价高昂的操作。
替代方法:使用 GSI
上述场景的扫描码:-
var docClient = new AWS.DynamoDB.DocumentClient();
var params = {
TableName: "users",
FilterExpression: "#user_status = :user_status_val",
ExpressionAttributeNames: {
"#user_status": "user_status",
},
ExpressionAttributeValues: { ":user_status_val": 'somestatus' }
};
docClient.scan(params, onScan);
var count = 0;
function onScan(err, data) {
if (err) {
console.error("Unable to scan the table. Error JSON:", JSON.stringify(err, null, 2));
} else {
console.log("Scan succeeded.");
data.Items.forEach(function(itemdata) {
console.log("Item :", ++count,JSON.stringify(itemdata));
});
// continue scanning if we have more items
if (typeof data.LastEvaluatedKey != "undefined") {
console.log("Scanning for more...");
params.ExclusiveStartKey = data.LastEvaluatedKey;
docClient.scan(params, onScan);
}
}
}
AWS 文档示例对我不起作用。 @Hank 方法成功了。
在 lambda 中使用处理程序:
const AWS = require('aws-sdk');
const docClient = new AWS.DynamoDB.DocumentClient({
// optional tuning - 50% faster(cold) / 20% faster(hot)
apiVersion: '2012-08-10',
sslEnabled: false,
paramValidation: false,
convertResponseTypes: false
});
const tableName = 'series';
exports.handler = async (event, context, callback) => {
let params = { TableName: tableName };
let scanResults = [];
let items;
do {
items = await docClient.scan(params).promise();
items.Items.forEach((item) => scanResults.push(item));
params.ExclusiveStartKey = items.LastEvaluatedKey;
} while (typeof items.LastEvaluatedKey != "undefined");
callback(null, scanResults);
};
使用承诺和异步
const aws = require('aws-sdk');
aws.config.update({ region: 'us-east-1' });
const documentClient = new aws.DynamoDB.DocumentClient();
const scanAll = async (params) => {
let lastEvaluatedKey = 'dummy'; // string must not be empty
const itemsAll = [];
while (lastEvaluatedKey) {
const data = await documentClient.scan(params).promise();
itemsAll.push(...data.Items);
lastEvaluatedKey = data.LastEvaluatedKey;
if (lastEvaluatedKey) {
params.ExclusiveStartKey = lastEvaluatedKey;
}
}
return itemsAll;
}
像这样使用
const itemsAll = scanAll(params);
查询代码相同(只需将扫描替换为查询)
const AWS = require('aws-sdk');
const docClient = new AWS.DynamoDB.DocumentClient({
// optional tuning - 50% faster(cold) / 20% faster(hot)
apiVersion: '2012-08-10',
sslEnabled: false,
paramValidation: false,
convertResponseTypes: false,
region: 'us-east-2' // put your region
});
const tableName = 'tableName'; // put your tablename
exports.handler = async (event, context, callback) => {
let params = { TableName: tableName };
let scanResults = [];
let items;
do {
items = await docClient.scan(params).promise();
items.Items.forEach((item) => scanResults.push(item));
params.ExclusiveStartKey = items.LastEvaluatedKey;
} while (typeof items.LastEvaluatedKey != "undefined");
callback(null, scanResults);
};
我使用这样的承诺:
let AWS = require('aws-sdk');
let docClient = new AWS.DynamoDB.DocumentClient();
async function dbRead(params) {
let promise = docClient.scan(params).promise();
let result = await promise;
let data = result.Items;
if (result.LastEvaluatedKey) {
params.ExclusiveStartKey = result.LastEvaluatedKey;
data = data.concat(await dbRead(params));
}
return data;
}
并使用它:
let params = {
TableName: 'Table'
};
let data = await dbRead(params);
一个以JSON格式返回数据的节点快递解决方案:
let datapack=[];
item = {
TableName: ddbTable,
FilterExpression: "aws = :e AND begins_with ( Id, :t )",
ExpressionAttributeValues: {
":t" : "contact",
":e" : aws
},
ProjectionExpression: "Id,FirstName,LastName,cEmail",
};
docClient.scan(item, onScan);
function onScan(err, data) {
if (err) {
console.error("Unable to scan the table. Error JSON:", JSON.stringify(err, null, 2));
} else {
datapack = datapack.concat(data.Items);
});
if (typeof data.LastEvaluatedKey != "undefined") {
item.ExclusiveStartKey = data.LastEvaluatedKey;
docClient.scan(item, onScan);
} else {
res.json(datapack);
}
}
}
这是一个答案,它将提供索引结果,而不是使用昂贵的扫描,也是 JSON 格式的节点/快递。 注意 docClient.query 的使用:
datapack=[];
item = {
TableName: ddbTable,
IndexName: "cEmailIndex",
KeyConditionExpression : "aws = :e AND begins_with ( cEmail, :t )",
ExpressionAttributeValues: {
":t" : search,
":e" : aws
},
ProjectionExpression: "Id,FirstName,LastName,cEmail",
};
docClient.query(item, onScan);
function onScan(err, data) {
if (err) {
console.error("Unable to scan the table. Error JSON:", JSON.stringify(err, null, 2));
} else {
datapack = datapack.concat(data.Items);
if (typeof data.LastEvaluatedKey != "undefined") {
item.ExclusiveStartKey = data.LastEvaluatedKey;
docClient.query(item, onScan);
} else {
// console.log(JSON.stringify(datapack));
res.json(datapack);
}
}
}
您可以使用@aws/dynamodb-query-iterator 中的 ScanPaginator :
import { ScanPaginator } from '@aws/dynamodb-query-iterator';
import DynamoDB = require('aws-sdk/clients/dynamodb');
const paginator = new ScanPaginator(
new DynamoDB.DocumentClient(),
{
TableName: "users",
FilterExpression: "#user_status = :user_status_val",
ExpressionAttributeNames: {
"#user_status": "user_status",
},
ExpressionAttributeValues: { ":user_status_val": 'somestatus' }
}
);
for await (const page of paginator) {
// do something with `page`, e.g. myFunction(page.Items)
}
对于那些不使用 AWS.DynamoDB.DocumentClient 的人,此解决方案将起作用。 为了便于阅读和使用 async/await,我已将功能拆分为多个模块。
const AWS = require("aws-sdk");
AWS.config.update({
// update table region here
region: "us-west-2"
});
var dynamodb = new AWS.DynamoDB();
const performAsynScanOperation = (scanParams) => {
return new Promise((resolve, reject) => {
dynamodb.scan(scanParams, function (err, responseData) {
if (err) {
reject(err)
} else {
resolve(responseData)
}
})
})
}
const getAllRecords = async (tableName) => {
let allItems = [];
let LastEvaluatedKeyFlag = true;
let scanParams = { TableName: tableName }
while (LastEvaluatedKeyFlag) {
let responseData = await performAsynScanOperation(scanParams)
let batchItems = responseData.Items;
allItems = allItems.concat(batchItems);
if (responseData.LastEvaluatedKey) {
LastEvaluatedKeyFlag = true;
console.log('LastEvaluatedKey', responseData.LastEvaluatedKey)
scanParams.ExclusiveStartKey = responseData.LastEvaluatedKey
} else {
LastEvaluatedKeyFlag = false;
}
}
return allItems;
}
getAllRecords('<Name of table>').then((allItems)=>{
console.log(allItems)
})
@大卫,
我试图使您的示例工作(JSON)
// Setting up our dependencies
const AWS = require('aws-sdk');
const https = require('https');
const dynamodb = new AWS.DynamoDB.DocumentClient();
// Creating an eviornment variable to store DynamoDB table name
const TABLE_NAME = process.env.table_name;
exports.handler = function(event, context, callback) {
var siteNum = "7261"; // event.siteNbr;
var res;
let datapack =[];
let item = {
TableName:TABLE_NAME,
IndexName: "SiteNumIndex",
KeyConditionExpression: "SiteNum = :siteNum",
"ExpressionAttributeValues": {
":siteNum": siteNum
}
};
console.log("siteNbr: " + siteNum) ;
dynamodb.query(item, onScan);
function onScan(err, data) {
if (err) {
console.error("Unable to scan the table. Error JSON:", JSON.stringify(err, null, 2));
} else {
datapack = datapack.concat(data.Items);
if (typeof data.LastEvaluatedKey != "undefined") {
item, onScan.ExclusiveStartKey = data.LastEvaluatedKey;
dynamodb.query(item, onScan);
} else {
console.log(JSON.stringify(datapack));
//data.json(datapack);
}
}
}
};
最后一个console.log确实编写了JSON,但是响应返回null。 请原谅我的无知,我还不是node.js开发人员。 在您的示例中,什么是res? 它从何而来? 我习惯于C#,在其中我会做类似“返回JSON.stringify(datapack);”的操作
这是扫描所有记录的替代品:
const scanAll = async (params) => {
let all = [];
while (true) {
let data = await new Promise((resolve, reject) => {
db.scan(params, function (err, data) {
if (err)
reject(err);
else
resolve(data);
});
});
all = all.concat(data.Items);
if (data.LastEvaluatedKey)
params.ExclusiveStartKey = data.LastEvaluatedKey;
else
break;
}
return all;
};
用法:
scanAll(query)
.catch((err) => {
})
.then((records) => {
});
}
scan 方法读取表中的每一项并返回表中的所有数据。 您可以提供一个可选的 filter_expression,以便只返回与您的条件匹配的项目。 但是,只有在扫描了整个表之后才应用过滤器。 参考
我正在分享重构的onScan
函数希望它onScan
帮助。
var AWS = require("aws-sdk");
AWS.config.update({
region: "us-west-2",
endpoint: "http://localhost:8000"
});
var docClient = new AWS.DynamoDB.DocumentClient();
async function read() {
const params = {
TableName: "tableName"
// options can be passed here e.g.
// FilterExpression: "#yr between :start_yr and :end_yr",
};
let items = [];
return new Promise((resolve, reject) => {
function onScan(err, data) {
if (err) {
console.error("Unable to scan the table. Error JSON:", JSON.stringify(err, null, 2));
reject();
} else {
items = items.concat(data.Items);
// continue scanning if we have more items, because
// scan can retrieve a maximum of 1MB of data
if (typeof data.LastEvaluatedKey !== "undefined") {
params.ExclusiveStartKey = data.LastEvaluatedKey;
docClient.scan(params, onScan);
} else {
resolve(items);
}
}
}
docClient.scan(params, onScan);
});
}
声明:本站的技术帖子网页,遵循CC BY-SA 4.0协议,如果您需要转载,请注明本站网址或者原文地址。任何问题请咨询:yoyou2525@163.com.