使用 logstash 拆分 Kafka 主题

Question

I have a kafka topic with several JSON schema's running through it from a few log sources. Is it possible to use logstash to sort messages into new topics based on the fields present in the incoming kafka messages?

Answer 1

see exmple below

input {
  kafka {
    id => "filter-detection-id"
    type => "filter_detections"
    topics => ["${KAFKA_TOPIC_NAME}"]
    group_id => "filter-detections-group"
    auto_offset_reset => "earliest"
    bootstrap_servers => "${KAFKA_URL}"
    enable_auto_commit => "false"
    client_id => "${MY_POD_NAME}"
  }
}

filter {
    json {
        skip_on_invalid_json => true
        source => "message"
        target => "message"
    }

    if "Suspicious Login" in [message][control_name] and [message][fl_metadata][anomaly_detection][severity] in ["Critical", "High"] {
        mutate { add_field => { "[@metadata][detection_type]" => "Suspicious Login" } }
    }

    if "Phished" in [message][control_name] {
        mutate { add_field => { "[@metadata][detection_type]" => "Phished" } }
    }
}

output {
    if [@metadata][detection_type] == "Suspicious Login" {
        kafka {
            bootstrap_servers => "${KAFKA_URL}"
            topic_id => "${KAFKA_USERSSUSPICIOUSLOGINDETECTIONS_NAME}"
            client_id => "${MY_POD_NAME}.${KAFKA_USERSSUSPICIOUSLOGINDETECTIONS_NAME}"
            codec => "json"
        }
    }
    if [@metadata][detection_type] == "Phished" {
        kafka {
            bootstrap_servers => "${KAFKA_URL}"
            topic_id => "${KAFKA_USERSISPHISHEDDETECTIONS_NAME}"
            client_id => "${MY_POD_NAME}.${KAFKA_USERSISPHISHEDDETECTIONS_NAME}"
            codec => "json"
        }
    }
}