比较elasticsearch索引中不同的文档字段值

Question

只是想知道如何比较 2 个不同文档的值。 我正在索引中摄取以下值。

我希望查询和比较每个 plugin_instance 的“allocated-mb”>“max-mb”的每个文档字段“type_instance”值。

POST test/_bulk
{"index":{"_id":1}}
{"plugin_instance": "root-yarn-queue-name-1", "type_instance": "allocated-mb", "value": 4024}
{"index":{"_id":2}}
{ "plugin_instance": "root-yarn-queue-name-1", "type_instance": "max-mb", "value": 2048}
{"index":{"_id":3}}
{"plugin_instance": "root-yarn-queue-name-2", "type_instance": "max-mb", "value": 3048}
{"index":{"_id":4}}
{"root-yarn-queue-name-2", "type_instance": "allocated-mb", "value": 1028}
{"index":{"_id":5}}
{"plugin_instance": "some-random-queue-name-2", "type_instance": "allocated-mb", "value": 2028}
{"index":{"_id":6}}
{"plugin_instance": "some-random-queue-name-2", "type_instance": "max-mb", "value": 2028}

---

只是想知道实现以下目标的简单方法是什么

1. 选择带有 plugin_instance=root-yarn-queue-name-* 的记录
2. 选择带有 type_instance 的记录（allocated-mb、located-vcores、max-mb、max-vcores）
3. 将具有相同 plugin_instance 的记录分组（获取存储桶中队列的记录）
4. 比较分配的 mb > max-mb 和分配的 vcore > max-vcore 的值，并选择在给定时间段内满足所有这些条件的那些记录

到目前为止，我已经设法根据 plugin_instance 对文档进行存储。 想知道根据“type_instance”比较每个存储桶的文档的简单方法是什么

GET test/_search
{
  "query": {
    "query_string": {
      "fields": [
        "plugin_instance.keyword",
        "type_instance.keyword"
        ],
      "query": "root-yarn-queue-name-* AND (max-mb OR allocated-mb)"
    }
  },
    "aggs": {
    "byField": {
      "terms": {
        "field": "plugin_instance.keyword"
      }
    }
  }
}

Answer 1

解决上述问题的一种方法是在客户端获取文档和执行过滤。 第二个将涉及使用bucket_selector 聚合

映射

{
  "index60" : {
    "mappings" : {
      "properties" : {
        "plugin_instance" : {
          "type" : "text",
          "fields" : {
            "keyword" : {
              "type" : "keyword",
              "ignore_above" : 256
            }
          }
        },
        "type_instance" : {
          "type" : "text",
          "fields" : {
            "keyword" : {
              "type" : "keyword",
              "ignore_above" : 256
            }
          }
        },
        "value" : {
          "type" : "long"
        }
      }
    }
  }
}

数据：

  "hits" : [
      {
        "_index" : "index60",
        "_type" : "_doc",
        "_id" : "vCN--3AB_Wo5Rvhl5c7f",
        "_score" : 1.0,
        "_source" : {
          "plugin_instance" : "root-yarn-queue-name-1",
          "type_instance" : "max-vcore",
          "value" : 1000
        }
      },
      {
        "_index" : "index60",
        "_type" : "_doc",
        "_id" : "vSN--3AB_Wo5Rvhl9M6R",
        "_score" : 1.0,
        "_source" : {
          "plugin_instance" : "root-yarn-queue-name-1",
          "type_instance" : "max-mb",
          "value" : 2048
        }
      },
      {
        "_index" : "index60",
        "_type" : "_doc",
        "_id" : "viN--3AB_Wo5Rvhl_s5m",
        "_score" : 1.0,
        "_source" : {
          "plugin_instance" : "root-yarn-queue-name-2",
          "type_instance" : "max-mb",
          "value" : 3048
        }
      },
      {
        "_index" : "index60",
        "_type" : "_doc",
        "_id" : "wCN_-3AB_Wo5Rvhlhc53",
        "_score" : 1.0,
        "_source" : {
          "plugin_instance" : "root-yarn-queue-name-2",
          "type_instance" : "allocated-mb",
          "value" : 1028
        }
      },
      {
        "_index" : "index60",
        "_type" : "_doc",
        "_id" : "wSOA-3AB_Wo5RvhlCc5r",
        "_score" : 1.0,
        "_source" : {
          "plugin_instance" : "root-yarn-queue-name-1",
          "type_instance" : "allocated-mb",
          "value" : 3000
        }
      }
    ]

询问：

1. 创建一个桶 root-yarn-queue-name-.*
2. 创建 ["max-mb","allocated-mb"] 的子桶
3. 找到 ["max-mb","allocated-mb"] 的最大值
4. 查找分配的 mb 的值
5. 选择一个已分配-mb 具有最大值的存储桶

{
  "size": 0,
  "aggs": {
    "plugin_instance": {
      "terms": {
        "field": "plugin_instance.keyword",
        "include": "root-yarn-queue-name-.*", --> pattern
        "size": 10000
      },
      "aggs": {
        "instance": {
          "terms": {
            "field": "type_instance.keyword",
            "include": [  --> create a bucket of these two
              "max-mb",
              "allocated-mb"
            ],
            "size": 10
          }
        },
        "maxValue": {
          "max": {
            "field": "value"
          }
        },
        "allocated-mb": {
          "filter": {
            "term": {
              "type_instance.keyword": "allocated-mb"
            }
          },
          "aggs": {
            "filtered_maxValue": {
              "max": {
                "field": "value"
              }
            }
          }
        },
        "my_bucket": {
          "bucket_selector": {
            "buckets_path": {
              "filteredValue": "allocated-mb>filtered_maxValue",
              "maxValue": "maxValue"
            },
            "script": "params.filteredValue==params.maxValue"
          }
        }
      }
    }
  }
}

结果：

 "aggregations" : {
    "plugin_instance" : {
      "doc_count_error_upper_bound" : 0,
      "sum_other_doc_count" : 0,
      "buckets" : [
        {
          "key" : "root-yarn-queue-name-1",
          "doc_count" : 3,
          "instance" : {
            "doc_count_error_upper_bound" : 0,
            "sum_other_doc_count" : 0,
            "buckets" : [
              {
                "key" : "allocated-mb",
                "doc_count" : 1
              },
              {
                "key" : "max-mb",
                "doc_count" : 1
              }
            ]
          },
          "maxValue" : {
            "value" : 3000.0
          },
          "allocated-mb" : {
            "doc_count" : 1,
            "filtered_maxValue" : {
              "value" : 3000.0
            }
          }
        }
      ]
    }
  }

如果您对此有任何疑问，请告诉我