[英]How to use the REST API for Activity Runs in Azure Data Factory?
I am trying to get all the information about the activities in a pipeline and log it into some SQL Database.我正在尝试获取有关管道中活动的所有信息并将其记录到一些 SQL 数据库中。
I did some R&D and I found that I can do it with RESP API.我做了一些研发,发现用 RESP API 可以做到。
I read and performed it with the help of below link:-我在以下链接的帮助下阅读并执行了它:-
https://docs.microsoft.com/en-us/rest/api/datafactory/activity-runs/query-by-pipeline-run#runqueryfilter https://docs.microsoft.com/en-us/rest/api/datafactory/activity-runs/query-by-pipeline-run#runqueryfilter
But the problem is, I am passing the correct subscription Id , run Id , and the lastUpdatedAfter , lastUpdatedBefore .但问题是,我传递了正确的订阅 Id 、运行 Id和lastUpdatedAfter 、 lastUpdatedBefore 。 But I am getting a empty array in the response of the REST API POST
method.但是我在REST API POST
方法的响应中得到一个空数组。
Also tell me, If I am missing something in REQUEST BODY .还告诉我,如果我在REQUEST BODY中遗漏了什么。
Here are the ScreenShots :-以下是截图:-
URL for the API URL 用于 API
How are you getting the Pipeline run ID, make sure that is right one and runs exist between the interval specified (UTC).你如何获得管道运行 ID,确保它是正确的,并且运行存在于指定的间隔 (UTC) 之间。 Format the JSON body in the request as I've shown.如我所示,在请求中格式化 JSON 正文。 Validate it using some tool or editor.使用某些工具或编辑器对其进行验证。 And use the right authentication.并使用正确的身份验证。
Are you sure you are not seeing any error at all?你确定你没有看到任何错误吗?
Activity Runs - Query By Pipeline Run 活动运行 - 按管道运行查询
https://management.azure.com/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.DataFactory/factories/{factoryName}/pipelineruns/{runId}/queryActivityruns?api-version=2018-06-01
Output
{
"value": [
{
"activityRunEnd": "2021-10-12T13:03:17.1874105Z",
"activityName": "Data flow1",
"activityRunStart": "2021-10-12T13:00:14.7523815Z",
"activityType": "ExecuteDataFlow",
"durationInMs": 182435,
"retryAttempt": null,
"error": {
"errorCode": "",
"message": "",
"failureType": "",
"target": "Data flow1",
"details": ""
},
"activityRunId": "5e7aece7-5fb7-4ffd-939e-d01d3fcb494a",
"iterationHash": "",
"input": {
"dataflow": {
"referenceName": "generateParquet",
"type": "DataFlowReference",
"parameters": {},
"datasetParameters": {
"source1": {},
"sink1": {}
}
},
"staging": {},
"compute": {
"coreCount": 8,
"computeType": "General"
},
"traceLevel": "Fine",
"dataFlowETag": "48019f9b-0000-0300-0000-616475e20000"
},
"linkedServiceName": "",
"output": {
"runStatus": {
"computeAcquisitionDuration": 107335,
"version": "20211011.2",
"profile": {
"source1": {
"computed": [],
"lineage": {},
"dropped": 0,
"drifted": 0,
"newer": 16,
"total": 16,
"updated": 0
},
"sink1": {
"computed": [],
"lineage": {
"CompanyName": {
"mapped": true,
"from": [
{
"source": "source1",
"columns": [
"CompanyName"
]
}
]
},
"LastName": {
"mapped": true,
"from": [
{
"source": "source1",
"columns": [
"LastName"
]
}
]
},
"MiddleName": {
"mapped": true,
"from": [
{
"source": "source1",
"columns": [
"MiddleName"
]
}
]
},
"PasswordSalt": {
"mapped": true,
"from": [
{
"source": "source1",
"columns": [
"PasswordSalt"
]
}
]
},
"ModifiedDate": {
"mapped": true,
"from": [
{
"source": "source1",
"columns": [
"ModifiedDate"
]
}
]
},
"path": {
"mapped": true,
"from": [
{
"source": "source1",
"columns": [
"path"
]
}
]
},
"rowguid": {
"mapped": true,
"from": [
{
"source": "source1",
"columns": [
"rowguid"
]
}
]
},
"Title": {
"mapped": true,
"from": [
{
"source": "source1",
"columns": [
"Title"
]
}
]
},
"NameStyle": {
"mapped": true,
"from": [
{
"source": "source1",
"columns": [
"NameStyle"
]
}
]
},
"CustomerID": {
"mapped": true,
"from": [
{
"source": "source1",
"columns": [
"CustomerID"
]
}
]
},
"FirstName": {
"mapped": true,
"from": [
{
"source": "source1",
"columns": [
"FirstName"
]
}
]
},
"Suffix": {
"mapped": true,
"from": [
{
"source": "source1",
"columns": [
"Suffix"
]
}
]
},
"Phone": {
"mapped": true,
"from": [
{
"source": "source1",
"columns": [
"Phone"
]
}
]
},
"PasswordHash": {
"mapped": true,
"from": [
{
"source": "source1",
"columns": [
"PasswordHash"
]
}
]
},
"SalesPerson": {
"mapped": true,
"from": [
{
"source": "source1",
"columns": [
"SalesPerson"
]
}
]
},
"EmailAddress": {
"mapped": true,
"from": [
{
"source": "source1",
"columns": [
"EmailAddress"
]
}
]
}
},
"dropped": 0,
"drifted": 0,
"newer": 0,
"total": 16,
"updated": 16
}
},
"metrics": {
"sink1": {
"format": "parquet",
"stages": [
{
"stage": 1,
"partitionTimes": [
525
],
"lastUpdateTime": "2021-10-12 13:02:50.453",
"bytesWritten": 0,
"bytesRead": 199491,
"partitionStatus": "Success",
"streams": {
"source1": {
"count": 847,
"cached": false,
"totalPartitions": 1,
"partitionStatus": "Success",
"partitionCounts": [
847
],
"type": "source"
}
},
"target": "sink1",
"time": 640,
"progressState": "Completed"
},
{
"stage": 2,
"partitionTimes": [
3436,
3373,
3441,
3475,
2304,
2321,
2364,
2291,
1989,
2435
],
"lastUpdateTime": "2021-10-12 13:02:58.897",
"bytesWritten": 177728,
"bytesRead": 0,
"partitionStatus": "Success",
"streams": {
"sink1": {
"count": 847,
"cached": false,
"totalPartitions": 10,
"partitionStatus": "Success",
"partitionCounts": [
84,
85,
85,
85,
85,
85,
85,
85,
84,
84
],
"type": "sink"
}
},
"target": "sink1",
"time": 8452,
"progressState": "Completed"
}
],
"sinkPostProcessingTime": 0,
"data": [],
"store": "blob",
"rowsWritten": 847,
"details": {
"postCommandsDuration": [
3
],
"preCommandsDuration": [
1
]
},
"sources": {
"source1": {
"rowsRead": 847,
"store": "blob",
"details": {
"fileSystemInitDuration": [
7751
]
},
"format": "delimited"
}
},
"sinkProcessingTime": 12380
}
},
"dsl": "\nsource() ~> source1\n\nsource1 sink() ~> sink1"
},
"effectiveIntegrationRuntime": "DefaultIntegrationRuntime (Central US)",
"billingReference": {
"activityType": "executedataflow",
"billableDuration": [
{
"meterType": "General",
"duration": 0.3691165737777778,
"unit": "coreHour",
"sessionType": "JobCluster"
}
]
},
"reportLineageToPurview": {
"status": "NotReported"
}
},
"userProperties": {},
"pipelineName": "partiion_parquets",
"pipelineRunId": "c619e850-c7de-4d1f-8d0c-4b73dcacb9d8",
"status": "Succeeded",
"recoveryStatus": "None",
"integrationRuntimeNames": [
"defaultintegrationruntime"
],
"executionDetails": {
"integrationRuntime": [
{
"name": "AutoResolveIntegrationRuntime",
"type": "Managed",
"computeType": "General",
"coreCount": "8"
}
]
},
"id": "/SUBSCRIPTIONS/B83C1EE3-xxxx-xxxx-B5BA-2E83A074C23K/RESOURCEGROUPS/myrg/PROVIDERS/MICROSOFT.DATAFACTORY/FACTORIES/KTESTADF/pipelineruns/c619e850-xxxx-xxxx-xxxx-4b73dcacb9d8/activityruns/5e7aece7-xxxx-4ffd-939e-d01d3fcb494a"
}
],
"ADFWebActivityResponseHeaders": {
"Pragma": "no-cache",
"Strict-Transport-Security": "max-age=31536000; includeSubDomains",
"X-Content-Type-Options": "nosniff",
"x-ms-ratelimit-remaining-subscription-reads": "11999",
"x-ms-request-id": "e8fa08f2-594a-4464-b43e-d71104c1950c",
"x-ms-correlation-request-id": "e8fa08f2-594a-4464-b43e-d71104c1950c",
"x-ms-routing-request-id": "CENTRALUS:20211012T131050Z:e8fa08f2-594a-4464-b43e-d71104c1950c",
"Cache-Control": "no-cache",
"Date": "Tue, 12 Oct 2021 13:10:49 GMT",
"Server": "Microsoft-IIS/10.0",
"X-Powered-By": "ASP.NET",
"Content-Length": "4307",
"Content-Type": "application/json; charset=utf-8",
"Expires": "-1"
},
"effectiveIntegrationRuntime": "DefaultIntegrationRuntime (Central US)",
"executionDuration": 0,
"durationInQueue": {
"integrationRuntimeQueue": 0
},
"billingReference": {
"activityType": "ExternalActivity",
"billableDuration": [
{
"meterType": "AzureIR",
"duration": 0.016666666666666666,
"unit": "Hours"
}
]
}
}
From the above you can filter for "activityType": "ExecuteDataFlow"
, durationInMs
, activityRunStart
, activityRunEnd
, invokedByType
, input
, output
and other metrics.从上面您可以过滤"activityType": "ExecuteDataFlow"
、 durationInMs
、 activityRunStart
、 activityRunEnd
、 invokedByType
、 input
、 output
和其他指标。
Just follow Azure Data Factory version 2 (V2) REST API operations , parameterize selected outputs and store in variables if necessary.只需遵循Azure 数据工厂版本 2 (V2) REST API 操作,参数化选定的输出并在必要时存储在变量中。 Using Add dynamic content
build URL and JSON request body appropriately.使用Add dynamic content
适当地构建 URL 和 JSON 请求正文。
声明:本站的技术帖子网页,遵循CC BY-SA 4.0协议,如果您需要转载,请注明本站网址或者原文地址。任何问题请咨询:yoyou2525@163.com.