get Response from AWS step function after execution of each map iteration

Question

I have an use case, where user is defined a workflow. As user hits the API, step function gets started. A lambda returns a list of steps based on users & start the workflow eg:

step1. it process(extracts) the doc,
step2. inserts some data to DB

step3. .....

step4. .....

here steps are in map, as these steps are returned from lambda. Is there any way to get response from each map iteration, so that I can show user at which step the process is, & the output of previous step.

here is my step function design:

test-fetch-input-workflow : returns an array: [step1, step2, ....]

workflow step-lambda : calls some lambda based on step name

Answer 1

Assuming that you are using Standard Workflows, you can get this information using the GetExecutionHistory API Action: https://docs.aws.amazon.com/step-functions/latest/apireference/API_GetExecutionHistory.html

You will want to look for events of type MapIterationSucceeded, grab the previousEventId, then look for the output in the HistoryEvent with that Id.

For example, if you had a state machine like this:

{    
"StartAt": "Generate Array for Map",
"States": {
  "Generate Array for Map": {
    "Type": "Pass",
    "Next": "Map",
    "Result": [
      1,
      2,
      3,
      4,
      5
    ]
  },
  "Map": {
    "Type": "Map",
    "ItemProcessor": {
      "ProcessorConfig": {
        "Mode": "INLINE"
      },
      "StartAt": "Generate Random Number",
      "States": {
        "Generate Random Number": {
          "Type": "Pass",
          "End": true,
          "Parameters": {
            "Random Number.$": "States.MathRandom(1,1000)"
          }
        }
      }
    },
    "End": true
  }
}

}

Your execution history would look like this:

{
"events": [
    {
        "timestamp": "2022-12-08T16:44:07.651000-08:00",
        "type": "ExecutionStarted",
        "id": 1,
        "previousEventId": 0,
        "executionStartedEventDetails": {
            "input": "{\n    \"Comment\": \"Insert your JSON here\"\n}",
            "inputDetails": {
                "truncated": false
            },
            "roleArn": "arn:aws:iam::000000000000:role/service-role/StepFunctions-MapTest-role-613230dd"
        }
    },
    {
        "timestamp": "2022-12-08T16:44:07.698000-08:00",
        "type": "PassStateEntered",
        "id": 2,
        "previousEventId": 0,
        "stateEnteredEventDetails": {
            "name": "Generate Array for Map",
            "input": "{\n    \"Comment\": \"Insert your JSON here\"\n}",
            "inputDetails": {
                "truncated": false
            }
        }
    },
    {
        "timestamp": "2022-12-08T16:44:07.698000-08:00",
        "type": "PassStateExited",
        "id": 3,
        "previousEventId": 2,
        "stateExitedEventDetails": {
            "name": "Generate Array for Map",
            "output": "[1,2,3,4,5]",
            "outputDetails": {
                "truncated": false
            }
        }
    },
    {
        "timestamp": "2022-12-08T16:44:07.698000-08:00",
        "type": "MapStateEntered",
        "id": 4,
        "previousEventId": 3,
        "stateEnteredEventDetails": {
            "name": "Map",
            "input": "[1,2,3,4,5]",
            "inputDetails": {
                "truncated": false
            }
        }
    },
    {
        "timestamp": "2022-12-08T16:44:07.698000-08:00",
        "type": "MapStateStarted",
        "id": 5,
        "previousEventId": 4,
        "mapStateStartedEventDetails": {
            "length": 5
        }
    },
    {
        "timestamp": "2022-12-08T16:44:07.698000-08:00",
        "type": "MapIterationStarted",
        "id": 6,
        "previousEventId": 5,
        "mapIterationStartedEventDetails": {
            "name": "Map",
            "index": 0
        }
    },
    {
        "timestamp": "2022-12-08T16:44:07.698000-08:00",
        "type": "MapIterationStarted",
        "id": 7,
        "previousEventId": 5,
        "mapIterationStartedEventDetails": {
            "name": "Map",
            "index": 1
        }
    },
    {
        "timestamp": "2022-12-08T16:44:07.698000-08:00",
        "type": "MapIterationStarted",
        "id": 8,
        "previousEventId": 5,
        "mapIterationStartedEventDetails": {
            "name": "Map",
            "index": 2
        }
    },
    {
        "timestamp": "2022-12-08T16:44:07.698000-08:00",
        "type": "MapIterationStarted",
        "id": 9,
        "previousEventId": 5,
        "mapIterationStartedEventDetails": {
            "name": "Map",
            "index": 3
        }
    },
    {
        "timestamp": "2022-12-08T16:44:07.698000-08:00",
        "type": "MapIterationStarted",
        "id": 10,
        "previousEventId": 5,
        "mapIterationStartedEventDetails": {
            "name": "Map",
            "index": 4
        }
    },
    {
        "timestamp": "2022-12-08T16:44:07.698000-08:00",
        "type": "PassStateEntered",
        "id": 11,
        "previousEventId": 6,
        "stateEnteredEventDetails": {
            "name": "Generate Random Number",
            "input": "1",
            "inputDetails": {
                "truncated": false
            }
        }
    },
    {
        "timestamp": "2022-12-08T16:44:07.698000-08:00",
        "type": "PassStateExited",
        "id": 12,
        "previousEventId": 11,
        "stateExitedEventDetails": {
            "name": "Generate Random Number",
            "output": "{\"Random Number\":259}",
            "outputDetails": {
                "truncated": false
            }
        }
    },
    {
        "timestamp": "2022-12-08T16:44:07.698000-08:00",
        "type": "MapIterationSucceeded",
        "id": 13,
        "previousEventId": 12,
        "mapIterationSucceededEventDetails": {
            "name": "Map",
            "index": 0
        }
    },
    {
        "timestamp": "2022-12-08T16:44:07.698000-08:00",
        "type": "PassStateEntered",
        "id": 14,
        "previousEventId": 7,
        "stateEnteredEventDetails": {
            "name": "Generate Random Number",
            "input": "2",
            "inputDetails": {
                "truncated": false
            }
        }
    },
    {
        "timestamp": "2022-12-08T16:44:07.698000-08:00",
        "type": "PassStateExited",
        "id": 15,
        "previousEventId": 14,
        "stateExitedEventDetails": {
            "name": "Generate Random Number",
            "output": "{\"Random Number\":776}",
            "outputDetails": {
                "truncated": false
            }
        }
    },
    {
        "timestamp": "2022-12-08T16:44:07.698000-08:00",
        "type": "MapIterationSucceeded",
        "id": 16,
        "previousEventId": 15,
        "mapIterationSucceededEventDetails": {
            "name": "Map",
            "index": 1
        }
    },
    {
        "timestamp": "2022-12-08T16:44:07.698000-08:00",
        "type": "PassStateEntered",
        "id": 17,
        "previousEventId": 8,
        "stateEnteredEventDetails": {
            "name": "Generate Random Number",
            "input": "3",
            "inputDetails": {
                "truncated": false
            }
        }
    },
    {
        "timestamp": "2022-12-08T16:44:07.698000-08:00",
        "type": "PassStateExited",
        "id": 18,
        "previousEventId": 17,
        "stateExitedEventDetails": {
            "name": "Generate Random Number",
            "output": "{\"Random Number\":743}",
            "outputDetails": {
                "truncated": false
            }
        }
    },
    {
        "timestamp": "2022-12-08T16:44:07.698000-08:00",
        "type": "MapIterationSucceeded",
        "id": 19,
        "previousEventId": 18,
        "mapIterationSucceededEventDetails": {
            "name": "Map",
            "index": 2
        }
    },
    {
        "timestamp": "2022-12-08T16:44:07.698000-08:00",
        "type": "PassStateEntered",
        "id": 20,
        "previousEventId": 9,
        "stateEnteredEventDetails": {
            "name": "Generate Random Number",
            "input": "4",
            "inputDetails": {
                "truncated": false
            }
        }
    },
    {
        "timestamp": "2022-12-08T16:44:07.698000-08:00",
        "type": "PassStateExited",
        "id": 21,
        "previousEventId": 20,
        "stateExitedEventDetails": {
            "name": "Generate Random Number",
            "output": "{\"Random Number\":790}",
            "outputDetails": {
                "truncated": false
            }
        }
    },
    {
        "timestamp": "2022-12-08T16:44:07.698000-08:00",
        "type": "MapIterationSucceeded",
        "id": 22,
        "previousEventId": 21,
        "mapIterationSucceededEventDetails": {
            "name": "Map",
            "index": 3
        }
    },
    {
        "timestamp": "2022-12-08T16:44:07.698000-08:00",
        "type": "PassStateEntered",
        "id": 23,
        "previousEventId": 10,
        "stateEnteredEventDetails": {
            "name": "Generate Random Number",
            "input": "5",
            "inputDetails": {
                "truncated": false
            }
        }
    },
    {
        "timestamp": "2022-12-08T16:44:07.698000-08:00",
        "type": "PassStateExited",
        "id": 24,
        "previousEventId": 23,
        "stateExitedEventDetails": {
            "name": "Generate Random Number",
            "output": "{\"Random Number\":632}",
            "outputDetails": {
                "truncated": false
            }
        }
    },
    {
        "timestamp": "2022-12-08T16:44:07.698000-08:00",
        "type": "MapIterationSucceeded",
        "id": 25,
        "previousEventId": 24,
        "mapIterationSucceededEventDetails": {
            "name": "Map",
            "index": 4
        }
    },
    {
        "timestamp": "2022-12-08T16:44:07.698000-08:00",
        "type": "MapStateSucceeded",
        "id": 26,
        "previousEventId": 25
    },
    {
        "timestamp": "2022-12-08T16:44:07.698000-08:00",
        "type": "MapStateExited",
        "id": 27,
        "previousEventId": 25,
        "stateExitedEventDetails": {
            "name": "Map",
            "output": "[{\"Random Number\":259},{\"Random Number\":776},{\"Random Number\":743},{\"Random Number\":790},{\"Random Number\":632}]",
            "outputDetails": {
                "truncated": false
            }
        }
    },
    {
        "timestamp": "2022-12-08T16:44:07.745000-08:00",
        "type": "ExecutionSucceeded",
        "id": 28,
        "previousEventId": 27,
        "executionSucceededEventDetails": {
            "output": "[{\"Random Number\":259},{\"Random Number\":776},{\"Random Number\":743},{\"Random Number\":790},{\"Random Number\":632}]",
            "outputDetails": {
                "truncated": false
            }
        }
    }
]

}