calling an AWS Lambda function asynchronously from python with async/await

Question

I have an AWS lambda function that I need to call asynchronously (fire and forget) and get the result back when it is done in a non blocking way.

loop = asyncio.get_event_loop()

async def f(payload):
    lambda_client = boto3.client('lambda')
    response = lambda_client.invoke(
    FunctionName='FUNC_NAME',
    InvocationType='RequestResponse',
    LogType='Tail',
    Payload=payload,
    Qualifier='$LATEST'
    )
    response_body = response['Payload']
    response_str = response_body.read().decode('utf-8')
    response_dict = eval(response_str)
    return response_dict


async def g():

    payload = json.dumps({
      "test_bucket": "MY_BUCKET",
      "test_key": "my_test_key.csv",
      "testpred_bucket": "MY_BUCKET",
      "testpred_key": "my_test_key_new.csv",
      "problem": "APROBLEM"
    })
    # Pause here and come back to g() when f() is ready
    r = await f(payload)
    print(r)

This works but this does not really serve the purpose of fire and forget . I understand somehow I need to use asyncio.ensure_future but if I do asyncio.ensure_future(f(payload)) , how do I capture the return value for the function f . I am new to python async and it is not clear.

Can anyone please suggest?

Answer 1

• This is how i have done in my aws lambda, I wanted to do 4 post request and then collect all the responses.

    loop = asyncio.get_event_loop()

    if loop.is_closed():
        loop = asyncio.new_event_loop()

    #The perform_traces method i do all the post method
    task = loop.create_task(perform_traces(payloads, message, contact_centre))
    unique_match, error = loop.run_until_complete(task)
    loop.close()

In the perform_trace method this is how i have used wait with session

    future_dds_responses = []

    async with aiohttp.ClientSession() as session:

        for payload in payloads:
            future_dds_responses.append(dds_async_trace(session, payload, contact_centre))

        dds_responses, pending = await asyncio.wait(future_dds_responses)

In dds_async_trace this is how i have done the post using the aiohttp.ClientSession session

        async with session.post(pds_url,
                                data=populated_template_payload,
                                headers=PDS_HEADERS,
                                ssl=ssl_context) as response:
            status_code = response.status

Answer 2

I was able to resolve this with asyncio.ensure_future

import boto3
import json
from flask import jsonify
import signal
import sys
import asyncio
import aiohttp
import json


def run():
    loop = asyncio.get_event_loop()
    loop.run_until_complete(run_job())


async def run_job():
    asyncio.ensure_future(launch_lambda())  # fire and forget async_foo()
    print('waiting for future ...')

async def launch_lambda():

    payload = json.dumps({
      "test_bucket": "my_bucket",
      "test_key": "my_test_key",
      "testpred_bucket": "my_bucket",
      "testpred_key": "my_pred_key",
      "problem": "APROBLEM"
    })
    result = await get_lambda_response(payload)
    print(result)

async def get_lambda_response(payload):
        lambda_client = boto3.client('lambda')
        response = lambda_client.invoke(
        FunctionName='FUNC_NAME',
        InvocationType='RequestResponse',
        LogType='Tail',
        Payload=payload,
        Qualifier='$LATEST'
        )
        response_body = response['Payload']
        response_str = response_body.read().decode('utf-8')
        response_dict = eval(response_str)
        return response_dict


run()

Answer 3

What you need is to set InvocationType to 'Event'

import boto3

lambda_client = boto3.client('lambda')
lambda_payload = {"name:"name","age":"age"}
lambda_client.invoke(FunctionName='myfunctionname', 
                     InvocationType='Event',
                     Payload=lambda_payload)

for more info see: https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/lambda.html#Lambda.Client.invoke