Dynamodb filter expression not returning all results

Question

I want to scan all items for last 7 days, so what I do is I generate timestamp for 7 days back and filter for timestamp greater than that value. But this scan is returning a few results.

See the following Javascript:

const daysBack = (days) => {
  let date = new Date();
  date.setDate(date.getDate() - days);
  return date.getTime() ; 
}


const params = {
  TableName: process.env.DYNAMODB_TABLE,
  FilterExpression: "#ts > :z",
  ExpressionAttributeNames:{
      "#ts": "timestamp"
  },
  ExpressionAttributeValues: {
      ":z": daysBack(7)
  },
};

dynamoDb.scan(params, (error, result) => {
  // ... 
}

Answer 1

It is because on a SCAN operation dynamoDb will only send data upto 1mb only . If the records you want are of size more than 1mb automatically pagination happens.

If you log your Result then you will find an attribute called LastEvaluatedKey if this attribute is present then you will have to make another call to get the remaining data. This call has to be implemented recursively and you have to stop it when LastEvaluatedKey attribute is not present.

Lets see this example where project data is been fetched recursively and the whole data is appended in the array and then send.

let getFromDb = function (params, callback) {
    params.ConsistentRead = true;
    let projectCollection = [];
    dynamodbclient.scan(params, onQuery);

    function onQuery(err, data) {
        const methodName = 'onQuery';
        if (err) {
            callback(err);
            log.error(err, {
                class: className,
                func: methodName
            });
        } else {
            for (let i = constant.LENGTH_ZERO; i < data.Items.length; i++) {
                projectCollection.push(data.Items[i]);
            }
            if (typeof data.LastEvaluatedKey !== 'undefined') {
                params.ExclusiveStartKey = data.LastEvaluatedKey;
                dynamodbclient.scan(params, onQuery);
            } else {
                callback(err, projectCollection); //recursive call
            }
        }
    }
};