在us-east-2中將DynamoDB導出到S3 AWS Data Pipeline

Question

我想將一個dynamodb表備份（然后導入）到S3。 dynamodb表位於us-east-2中，但這是AWS數據管道不支持的區域。 AWS文檔似乎表明這應該不成問題，但是我似乎無法讓數據管道在us-east-2中查找表。

這是我的數據管道的導出。 當我運行此程序時，在查找dynamodb表時收到“找不到資源錯誤”消息。 如果我在運行該數據管道的us-west-2中臨時創建一個具有相同名稱的表，則該工作有效，但從us-west-2中的表中而不是us-east-2中提取了數據。 有什么辦法讓這項工作從配置中指定的區域中拉出？

{
  "objects": [
    {
      "readThroughputPercent": "#{myDDBReadThroughputRatio}",
      "name": "DDBSourceTable",
      "id": "DDBSourceTable",
      "type": "DynamoDBDataNode",
      "region": "us-east-2",
      "tableName": "#{myDDBTableName}"
    },
    {
      "period": "6 Hours",
      "name": "Every 6 hours",
      "id": "DefaultSchedule",
      "type": "Schedule",
      "startAt": "FIRST_ACTIVATION_DATE_TIME"
    },
    {
      "bootstrapAction": "s3://us-west-2.elasticmapreduce/bootstrap-actions/configure-hadoop, --yarn-key-value,yarn.nodemanager.resource.memory-mb=11520,--yarn-key-value,yarn.scheduler.maximum-allocation-mb=11520,--yarn-key-value,yarn.scheduler.minimum-allocation-mb=1440,--yarn-key-value,yarn.app.mapreduce.am.resource.mb=2880,--mapred-key-value,mapreduce.map.memory.mb=5760,--mapred-key-value,mapreduce.map.java.opts=-Xmx4608M,--mapred-key-value,mapreduce.reduce.memory.mb=2880,--mapred-key-value,mapreduce.reduce.java.opts=-Xmx2304m,--mapred-key-value,mapreduce.map.speculative=false",
      "name": "EmrClusterForBackup",
      "coreInstanceCount": "1",
      "coreInstanceType": "m3.xlarge",
      "amiVersion": "3.9.0",
      "masterInstanceType": "m3.xlarge",
      "id": "EmrClusterForBackup",
      "region": "us-west-2",
      "type": "EmrCluster",
      "terminateAfter": "1 Hour"
    },
    {
      "directoryPath": "#{myOutputS3Loc}/#{format(@scheduledStartTime, 'YYYY-MM-dd-HH-mm-ss')}",
      "name": "S3BackupLocation",
      "id": "S3BackupLocation",
      "type": "S3DataNode"
    },
    {
      "output": {
        "ref": "S3BackupLocation"
      },
      "input": {
        "ref": "DDBSourceTable"
      },
      "maximumRetries": "2",
      "name": "TableBackupActivity",
      "step": "s3://dynamodb-emr-us-west-2/emr-ddb-storage-handler/2.1.0/emr-ddb-2.1.0.jar,org.apache.hadoop.dynamodb.tools.DynamoDbExport,#{output.directoryPath},#{input.tableName},#{input.readThroughputPercent}",
      "id": "TableBackupActivity",
      "runsOn": {
        "ref": "EmrClusterForBackup"
      },
      "type": "EmrActivity",
      "resizeClusterBeforeRunning": "true"
    },
    {
      "failureAndRerunMode": "CASCADE",
      "schedule": {
        "ref": "DefaultSchedule"
      },
      "resourceRole": "data_pipeline_etl_role",
      "pipelineLogUri": "s3://MY_S3_BUCKET/",
      "role": "data_pipeline_pipeline_role",
      "scheduleType": "cron",
      "name": "Default",
      "id": "Default"
    }
  ],
  "parameters": [
    {
      "description": "Output S3 folder",
      "id": "myOutputS3Loc",
      "type": "AWS::S3::ObjectKey"
    },
    {
      "description": "Source DynamoDB table name",
      "id": "myDDBTableName",
      "type": "String"
    },
    {
      "default": "0.25",
      "watermark": "Enter value between 0.1-1.0",
      "description": "DynamoDB read throughput ratio",
      "id": "myDDBReadThroughputRatio",
      "type": "Double"
    },
    {
      "default": "us-east-1",
      "watermark": "us-east-1",
      "description": "Region of the DynamoDB table",
      "id": "myDDBRegion",
      "type": "String"
    }
  ],
  "values": {
    "myDDBRegion": "us-east-2",
    "myDDBTableName": "prod--users",
    "myDDBReadThroughputRatio": "0.25",
    "myOutputS3Loc": "s3://MY_S3_BUCKET"
  }
}

Answer 1

同樣的問題，在us-east-2中找不到DynamoDB表

Answer 2

是一次性關閉還是您想要連續做的事情？ 您能否使用DynamoDB全局表在受支持的區域中復制該表，然后在備份完成后立即刪除該區域？

全局表復制是免費的。 您只需為復制表的啟動和運行期間的容量付費。

https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/GlobalTables.html