在同一个Logstash配置文件中使用多个Elsticsearch输出时，将忽略模板

Question

我写了一个Logstash配置文件，其中包含三个文件输入（所有这些都是日志文件），三个过滤器（每个都有不同的模式）和三个elasticsearch输出（每个输出转到不同的索引）。 每个索引根据其输入类型具有不同的模板，索引按周分区。

当所描述的配置文件运行时，会发生问题，索引模板将被忽略，并且不会对索引创建生效。

在这种情况下，模板不起作用：

input {
    file {
        path => ["/path/to/file.log"]
        start_position => "beginning"
        sincedb_path => "/dev/null"
        ignore_older => 0
        type => 'type_1'
        }
    file {
        path => "/path/to/file2.log"
        start_position => "beginning"
        sincedb_path => "/dev/null"
        ignore_older => 0
        type => 'type_2'
    }
    file {
        path => ["/path/to/file3.log"]
        start_position => "beginning"
        sincedb_path => "/dev/null"
        ignore_older => 0
        type => 'type_3'
    }
}
filter {
    if [type] == "type_1" {
        csv {
            columns => ["column1","column2","column3"]
                separator => "|"
        }
        date {
            match => [ "column3", "EEE MMM dd HH:mm:ss zzz yyyy" ]
            target => "@timestamp"
        }
        date {
            match => [ "column3", "EEE MMM dd HH:mm:ss zzz yyyy" ]
            target => "column3"
        }   
        mutate {
            remove_field => [ "message" ]
            remove_field => [ "path" ]
            remove_field => [ "host" ]
        }
    }
    if [type] == "type_2" {
        csv {
            columns => ["column1","column2",]
            separator => "|"
        }
        mutate {
            remove_field => [ "message" ]
            remove_field => [ "path" ]
            remove_field => [ "host" ]
            convert => { "column1" => "float" }
        }
        date {
            match => [ "column2", "EEE MMM dd HH:mm:ss zzz yyyy" ]
            target => "@timestamp"
        }
        date {
            match => [ "column2", "EEE MMM dd HH:mm:ss zzz yyyy" ]
            target => "column2"
        }
    }
    if [type] == "type_3" {
        csv {
            columns => ["column1","column2","column3","column4"]
            separator => "|"
        }
        mutate {
            remove_field => [ "message" ]
            remove_field => [ "path" ]
            remove_field => [ "host" ]
            convert => { "column3" => "float" }
            convert => { "column1" => "float" }
        }
        date {
            match => [ "column4", "EEE MMM dd HH:mm:ss zzz yyyy" ]
            target => "@timestamp"
        }
        date {
             match => [ "column4", "EEE MMM dd HH:mm:ss zzz yyyy" ]
             target => "column4"
        }
    }
}
output {
    if [type] == "type_1" {
        elasticsearch { 
            hosts => ["localhost:9200"]
            index => "type_1-%{+xxxx.ww}"
        }
    }
    if [type] == "type_2" {
        elasticsearch { 
            hosts => ["localhost:9200"]
            index => "type_2-%{+xxxx.ww}"
        }
    }
    if [type] == "type_3" {
        elasticsearch {
            hosts => ["localhost:9200"]
            index => "type_3-%{+xxxx.ww}"
        }
    }
}

相反，当单个配置文件用于每个输入文件类型，过滤器和弹性搜索输出时，模板工作正常。

模板在这里工作正常：

input {
    file {
        path => ["/path/to/file.log"]
        start_position => "beginning"
        sincedb_path => "/dev/null"
        ignore_older => 0
        }
}
filter {
    csv {
        columns => ["column1","column2","column3"]
            separator => "|"
    }
    date {
        match => [ "column3", "EEE MMM dd HH:mm:ss zzz yyyy" ]
        target => "@timestamp"
    }
    date {
        match => [ "column3", "EEE MMM dd HH:mm:ss zzz yyyy" ]
        target => "column3"
    }   
    mutate {
        remove_field => [ "message" ]
        remove_field => [ "path" ]
        remove_field => [ "host" ]
    }
}
output {
    elasticsearch { 
        hosts => ["localhost:9200"]
        index => "type_1-%{+xxxx.ww}"
    }
}

我已经在配置文件中使用了以下参数：

• template =>“file_name.json”
• template_overwrite =>“true”
• manage_template =>“true”
• template_name =>“template_name”

但他们没有帮助。

有没有人之前有这个错误？

（我正在使用elasticsearch 2.3.2和logstash 2.3.2）

任何帮助将不胜感激

---

我的模板

TYPE_1

 curl -X PUT 'localhost:9200/_template/type_1' -d '
    {
      "template": "type_1*", 
      "settings" : {
        "index" : {
          "refresh_interval" : "30s"
        }
      },
      "mappings": {
        "logs" : {
            "_all": {
              "enabled": false
            },
            "_source": {
              "enabled": false
            },
            "dynamic": "strict",
            "properties" : {
                 "column3" : {
                    "type" : "date",
                    "format" : "strict_date_optional_time||epoch_millis",
                        "norms": {
                            "enabled": false
                        }
                  },
                  "@timestamp" : {
                    "format" : "strict_date_optional_time||epoch_millis",
                    "type" : "date",
                                "norms": {
                                    "enabled": false
                                }
                  },
                  "column2" : {
                    "type" : "string",
                    "index": "not_analyzed"             
                  },
                  "column1" : {
                    "type" : "string",
                    "index": "not_analyzed"
                  },
                  "@version" : {
                    "type" : "string",
                        "norms": {
                            "enabled": false
                        }
                  }
             }
         }
      }
    }';

TYPE_2

 curl -X PUT 'localhost:9200/_template/type_2' -d '
    {
      "template": "type_2*", 
      "settings" : {
        "index" : {
          "refresh_interval" : "30s"
        }
      },
      "mappings": {
        "logs" : {
            "_all": {
              "enabled": false
            },
            "_source": {
              "enabled": false
            },
            "dynamic": "strict",
            "properties" : {
                 "column2" : {
                    "type" : "date",
                    "format" : "strict_date_optional_time||epoch_millis",
                        "norms": {
                            "enabled": false
                        }
                  },
                  "@timestamp" : {
                    "format" : "strict_date_optional_time||epoch_millis",
                    "type" : "date",
                                "norms": {
                                    "enabled": false
                                }
                  },
                  "column1" : {
                    "type" : "float",
                    "index": "not_analyzed"             
                  },
                  "@version" : {
                    "type" : "string",
                        "norms": {
                            "enabled": false
                        }
                  }
             }
         }
      }
    }';

TYPE_3

curl -X PUT 'localhost:9200/_template/type_3' -d '
{
  "template": "type_3*", 
  "settings" : {
    "index" : {
      "refresh_interval" : "30s"
    }
  },
  "mappings": {
    "logs" : {
        "_all": {
          "enabled": false
        },
        "_source": {
          "enabled": false
        },
        "dynamic": "strict",
        "properties" : {
             "column4" : {
                "type" : "date",
                "format" : "strict_date_optional_time||epoch_millis",
                    "norms": {
                        "enabled": false
                    }
              },
              "@timestamp" : {
                "format" : "strict_date_optional_time||epoch_millis",
                "type" : "date",
                            "norms": {
                                "enabled": false
                            }
              },
              "column3" : {
                "type" : "float",
                "index": "not_analyzed"             
              },
              "column2" : {
                "type" : "string",
                "index": "not_analyzed"             
              },
               "column1" : {
                "type" : "float",
                "index": "not_analyzed"             
              },
              "@version" : {
                "type" : "string",
                    "norms": {
                        "enabled": false
                    }
              }
         }
     }
  }
}';

Answer 1

我找到了解决方案

Losgstash配置文件使用type =>'type_1'来区分输入文件，filteres和Elasticsearch输出，还定义了模板类型和名为“type”的新字段。

我们在映射模板“logs”中使用了默认类型，并且我们没有考虑logstash配置文件中定义的'type_1'。 因此，模板被忽略了。

解决方案：更改模板映射中的类型问题得到解决。

例如：

curl -X PUT 'localhost:9200/_template/type_1' -d '
    {
      "template": "type_1*", 
      "settings" : {
        "index" : {
          "refresh_interval" : "30s"
        }
      },
      "mappings": {
        "type_1" : {
            "_all": {
              "enabled": false
            },
            "_source": {
              "enabled": false
            },
            "dynamic": "strict",
            "properties" : {
                 "column3" : {
                    "type" : "date",
                    "format" : "strict_date_optional_time||epoch_millis",
                        "norms": {
                            "enabled": false
                        }
                  },
                  "@timestamp" : {
                    "format" : "strict_date_optional_time||epoch_millis",
                    "type" : "date",
                                "norms": {
                                    "enabled": false
                                }
                  },
                  "column2" : {
                    "type" : "string",
                    "index": "not_analyzed"             
                  },
                  "column1" : {
                    "type" : "string",
                    "index": "not_analyzed"
                  },
                  "@version" : {
                    "type" : "string",
                        "norms": {
                            "enabled": false
                        }
                  }
             }
         }
      }
    }';