Kibana. Filtering records by matching values from another filtering

Question

I see such messages in kibana for a period of 5 seconds:

Date, Message, TraceId

Dec 10, 2020 @ 10:49:50.285 New request start http://somehost/path1   7ec708ab153e644f
Dec 10, 2020 @ 10:49:51.179 New request end http://somehost/path1     7ec708ab153e644f
Dec 10, 2020 @ 10:49:52.285 New request start http://somehost/path2   1e090982aeb026a3
Dec 10, 2020 @ 10:49:54.285 New request start http://somehost/path3   b880dfa9c4fd39ad
Dec 10, 2020 @ 10:49:53.179 New request end http://somehost/path3     b880dfa9c4fd39ad
Dec 10, 2020 @ 10:49:54.349 New request start http://somehost/path4   65184024b220dd0c

How can I filter records to see only "New request start" lines that do not have corresponding "New request end" matching by "traceId"?

For example, for the lines above I want to see the result:

Dec 10, 2020 @ 10:49:52.285 New request start http://somehost/path2   1e090982aeb026a3
Dec 10, 2020 @ 10:49:54.349 New request start http://somehost/path4   65184024b220dd0c

Answer 1

You can

1. group by traceID
2. take only one result sorted by date, OR filter 1 result with "start" in the field "message"

Here are some exemples:

{
  "size": 0,
  "aggs": {
    "group_by_trace": {
      "terms": {
        "field": "TraceId.keyword",
        "size": 10,
        "min_doc_count": 2
      },
      "aggs": {
        "startt_request": {
        "top_hits": {
          "sort": [
            {
              "date": {
                "order": "asc"
              }
            }
          ],
          "_source": {
            "includes": [
              "date",
              "message",
              "TraceId"
            ]
          },
          "size": 1
        }
        }
      }
    }
  }
}

And the response:

{
  "aggregations" : {
    "group_by_trace" : {
      "doc_count_error_upper_bound" : 0,
      "sum_other_doc_count" : 0,
      "buckets" : [
        {
          "key" : "7ec708ab153e644f",
          "doc_count" : 2,
          "startt_request" : {
            "hits" : {
              "total" : {
                "value" : 2,
                "relation" : "eq"
              },
              "max_score" : null,
              "hits" : [
                {
                  "_index" : "testlog",
                  "_type" : "_doc",
                  "_id" : "SOvlZXYBTUPHNNy0GTa-",
                  "_score" : null,
                  "_source" : {
                    "date" : "Dec 10, 2020 @ 10:49:50.285",
                    "TraceId" : "7ec708ab153e644f",
                    "message" : "New request start http://somehost/path1"
                  },
                  "sort" : [
                    "Dec 10, 2020 @ 10:49:50.285"
                  ]
                }
              ]
            }
          }
        },
        {
          "key" : "b880dfa9c4fd39ad",
          "doc_count" : 2,
          "startt_request" : {
            "hits" : {
              "total" : {
                "value" : 2,
                "relation" : "eq"
              },
              "max_score" : null,
              "hits" : [
                {
                  "_index" : "testlog",
                  "_type" : "_doc",
                  "_id" : "rqLlZXYBcOugy9Fj5LZp",
                  "_score" : null,
                  "_source" : {
                    "date" : "Dec 10, 2020 @ 10:49:54.285",
                    "TraceId" : "b880dfa9c4fd39ad",
                    "message" : "New request start http://somehost/path3"
                  },
                  "sort" : [
                    "Dec 10, 2020 @ 10:49:54.285"
                  ]
                }
              ]
            }
          }
        }
      ]
    }
  }
}

Or better, you can use a filter:

GET /_search?filter_path=aggregations.group_by_trace.buckets.start_messages.buckets.start.start_request.hits.hits
{
  "size": 0,
  "aggs": {
    "group_by_trace": {
      "terms": {
        "field": "TraceId.keyword",
        "size": 10,
        "min_doc_count": 2
      },
      "aggs": {
        "start_messages": {
          "filters": {
            "filters": {
              "start": {
                "match": {
                  "message": "start"
                }
              }
            }
          },
          "aggs": {
            "start_request": {
              "top_hits": {
                "_source": {
                  "includes": [
                    "date",
                    "message",
                    "TraceId"
                  ]
                },
                "size": 1
              }
            }
          }
        }
      }
    }
  }
}

And the response:

{
  "aggregations" : {
    "group_by_trace" : {
      "buckets" : [
        {
          "start_messages" : {
            "buckets" : {
              "start" : {
                "start_request" : {
                  "hits" : {
                    "hits" : [
                      {
                        "_index" : "testlog",
                        "_type" : "_doc",
                        "_id" : "SOvlZXYBTUPHNNy0GTa-",
                        "_score" : 1.0,
                        "_source" : {
                          "date" : "Dec 10, 2020 @ 10:49:50.285",
                          "TraceId" : "7ec708ab153e644f",
                          "message" : "New request start http://somehost/path1"
                        }
                      }
                    ]
                  }
                }
              }
            }
          }
        },
        {
          "start_messages" : {
            "buckets" : {
              "start" : {
                "start_request" : {
                  "hits" : {
                    "hits" : [
                      {
                        "_index" : "testlog",
                        "_type" : "_doc",
                        "_id" : "rqLlZXYBcOugy9Fj5LZp",
                        "_score" : 1.0,
                        "_source" : {
                          "date" : "Dec 10, 2020 @ 10:49:54.285",
                          "TraceId" : "b880dfa9c4fd39ad",
                          "message" : "New request start http://somehost/path3"
                        }
                      }
                    ]
                  }
                }
              }
            }
          }
        }
      ]
    }
  }
}