使用 logstash 拆分 Kafka 主题

Question

我有一个 kafka 主题，其中有几个 JSON 模式从几个日志源运行。 是否可以使用 logstash 根据传入的 kafka 消息中存在的字段将消息分类为新主题？

Answer 1

见下面的例子

input {
  kafka {
    id => "filter-detection-id"
    type => "filter_detections"
    topics => ["${KAFKA_TOPIC_NAME}"]
    group_id => "filter-detections-group"
    auto_offset_reset => "earliest"
    bootstrap_servers => "${KAFKA_URL}"
    enable_auto_commit => "false"
    client_id => "${MY_POD_NAME}"
  }
}

filter {
    json {
        skip_on_invalid_json => true
        source => "message"
        target => "message"
    }

    if "Suspicious Login" in [message][control_name] and [message][fl_metadata][anomaly_detection][severity] in ["Critical", "High"] {
        mutate { add_field => { "[@metadata][detection_type]" => "Suspicious Login" } }
    }

    if "Phished" in [message][control_name] {
        mutate { add_field => { "[@metadata][detection_type]" => "Phished" } }
    }
}

output {
    if [@metadata][detection_type] == "Suspicious Login" {
        kafka {
            bootstrap_servers => "${KAFKA_URL}"
            topic_id => "${KAFKA_USERSSUSPICIOUSLOGINDETECTIONS_NAME}"
            client_id => "${MY_POD_NAME}.${KAFKA_USERSSUSPICIOUSLOGINDETECTIONS_NAME}"
            codec => "json"
        }
    }
    if [@metadata][detection_type] == "Phished" {
        kafka {
            bootstrap_servers => "${KAFKA_URL}"
            topic_id => "${KAFKA_USERSISPHISHEDDETECTIONS_NAME}"
            client_id => "${MY_POD_NAME}.${KAFKA_USERSISPHISHEDDETECTIONS_NAME}"
            codec => "json"
        }
    }
}