数据湖 AWS 无服务器 Amazon S3

Question

I trying to build a serverless data lake with Amazon Simple Storage Service (Amazon S3) as the primary data store. Ingested data lands in an Amazon S3 bucket that we refer to as the raw zone. To make that data available, I have to catalog its schema in the AWS Glue Data Catalog.

I do this using an AWS Lambda function invoked by an Amazon S3 trigger to start an AWS Glue crawler that catalogs the data.

When the crawler is finished creating the table definition, is invoke a second Lambda function using an Amazon CloudWatch Events rule. This step starts an AWS Glue ETL job to process and output the data into another Amazon S3 bucket that we refer to as the processed zone. The AWS Glue ETL job converts the data to Apache Parquet format and stores it in the processed S3 bucket

Lambda to run crawler:

var AWS = require('aws-sdk');
var glue = new AWS.Glue();
var sqs = new AWS.SQS();
    exports.handler = function(event, context,callback) {
        console.log(JSON.stringify(event, null, 3));
        if(event.Records.length > 0 && event.Records[0].eventSource == 'aws:sqs'){
            startCrawler('datacrawler', function(err2,data2){
                if(err2) callback(err2)
                else callback(null,data2)
            })
        }else{
        var dbName = 'datacatalog';
        var params = {
            DatabaseInput: {
                Name: dbName,
                Description: 'Rede Post database',
            }
        };
        glue.createDatabase(params, function(err, data) {
                var params1 = {
                    DatabaseName: dbName,
                    Name: 'datacrawler',
                    Role: 'service-role/rede-data-lake-GlueLabRole-1OI9OXN93676F',
                    Targets: {
                        S3Targets: [{ Path: 's3://rede-data-lake-raws3bucket-1qgllh1leebin/' }]
                    },
                    Description: 'crawler test'
                };
                glue.createCrawler(params1, function(err1, data1) {
                    startCrawler('datacrawler', function(err2,data2){
                        if(err2) callback(err2)
                        else callback(null,data2)
                    })
                });
        });
    };
};
function startCrawler(name,callback){
    var params = {
        Name: name,
    };
    glue.startCrawler(params, function(err, data) {
        if (err){
            console.log(JSON.stringify(err,null,3 ))
            var params1 = {
                MessageBody: 'retry',
                QueueUrl: 'https://sqs.us-east-2.amazonaws.com/094381036356/rede-data-lake-SQSqueue-1AWGW0PCYANIY'
            };
            sqs.sendMessage(params1, function(err1, data1) {
                if (err1) callback(err1);
                else     callback(null, data1)
            });
        }
        else{
            callback(null, data)
        }
    });
    }

Cloud Watch Event rule:

{
  "detail-type": [
    "Glue Crawler State Change"
  ],
  "source": [
    "aws.glue"
  ],
  "detail": {
    "crawlerName": [
      "datacrawler"
    ],
    "state": [
      "Succeeded"
    ]
  }
}

Lambda To Run the Glue Job:

var AWS = require('aws-sdk');
var sns = new AWS.SNS( { region: "us-east-2" });
var s3 = new AWS.S3();
var glue = new AWS.Glue({apiVersion: '2017-03-31'});
    exports.handler = function(event, context, callback) {
    console.log(JSON.stringify(event, null, 3));
        var params = {
            JobName: 'GlueSalesJob',
            Timeout: 20,
        };
        glue.startJobRun(params, function(err1, data1) {
            if (err1) {
                console.log(err1, err1.stack);}
            else {
                console.log(data1);}
        });
        console.log(JSON.stringify(event, null, 3));
    };

All working just fine when we deal with just one file and one glue job, I don't see how to scale it.

Imagine that I have various different file arrive at raw zone, each file into a folder, for each one I have to run the AWS Glue crawler and the AWS Glue ETL job and stores it in one folder inside the processed zone bucket.

Ex: SaleFile, installmentsFile, DebitFiles and etc…

How could I call the second lambda passing the name of the Job that should run for each file? Basically I need to identify the The file or the folder Ingested to call the appropriated Glue Job.

Someone could help me to find a solution for this? I appreciate any help. I'm very new with Amazon.

Answer 1

Good going: You are almost there :-)

When you listen to 'crawler state change event', you will get event object in lambda. It has following structure

"detail": {
    "crawlerName": "demo",
    ....
    ....
}

Use event["detail"]["crawlerName"] to get crawler name. Since each crawler is mapped with corresponding ETL job, you can start glue job by using this mapping.