How to view log in kibana

Question

I am new to ELK, I tried ELK Stack with springboot using net.logstash.logback.appender.LogstashTcpSocketAppender. I sent json messages to logstack. Below is my configuration -

logback-spring.xml

<configuration>
    <include resource="org/springframework/boot/logging/logback/defaults.xml" />
​   <springProperty scope="context" name="springAppName" source="spring.application.name" />

    <property name="LOG_FILE" value="./${springAppName}" />


    <property name="CONSOLE_LOG_PATTERN"
        value="%clr(%d{yyyy-MM-dd HH:mm:ss.SSS}){faint} %clr(${LOG_LEVEL_PATTERN:-%5p}) %clr(${PID:- }){magenta} %clr(---){faint} %clr([%15.15t]){faint} %clr(%-40.40logger{39}){cyan} %clr(:){faint} %m%n${LOG_EXCEPTION_CONVERSION_WORD:-%wEx}" />


    <appender name="logstash2"
        class="net.logstash.logback.appender.LogstashTcpSocketAppender">
        <destination>localhost:5000</destination>
        <encoder
            class="net.logstash.logback.encoder.LoggingEventCompositeJsonEncoder">
            `
            <providers>
                <timestamp>
                    <timeZone>UTC</timeZone>
                </timestamp>
                <pattern>
                    <pattern>
                        {
                        "severity": "%level",
                        "service": "${springAppName:-}",
                        "trace": "%X{X-B3-TraceId:-}",
                        "span": "%X{X-B3-SpanId:-}",
                        "parent": "%X{X-B3-ParentSpanId:-}",
                        "exportable":
                        "%X{X-Span-Export:-}",
                        "pid": "${PID:-}",
                        "thread": "%thread",
                        "class": "%logger{40}",
                        "rest": "%message"
                        }
                    </pattern>
                </pattern>
            </providers>
        </encoder>
        <keepAliveDuration>5 minutes</keepAliveDuration>
    </appender>
    ​
    <root level="INFO">
        <appender-ref ref="logstash" />
    </root>
</configuration>

config.json

input{
    tcp{
        port=> 5000
        host=> localhost
    }
}   

filter {
       # pattern matching logback pattern
       grok {
              match => { "message" => "%{TIMESTAMP_ISO8601:timestamp}\s+%{LOGLEVEL:severity}\s+\[%{DATA:service},%{DATA:trace},%{DATA:span},%{DATA:exportable}\]\s+%{DATA:pid}\s+---\s+\[%{DATA:thread}\]\s+%{DATA:class}\s+:\s+%{GREEDYDATA:rest}" }
       }
}

output {
    elasticsearch { hosts => ["localhost:9200"] }
}   

But when I open kibana to see the messages, I see whole log as message. like below-

Can some one help me achieving the output as below -

Answer 1

Your filter block should look like that:

filter {
       # pattern matching logback pattern
       grok {
              match => { "message" => "%{TIMESTAMP_ISO8601:timestamp}\s+%{LOGLEVEL:severity}\s+\[%{DATA:service},%{DATA:trace},%{DATA:span},%{DATA:exportable}\]\s+%{DATA:pid}\s+---\s+\[%{DATA:thread}\]\s+%{DATA:class}\s+:\s+%{GREEDYDATA:rest}" }
       }
       json{
              source => "message"
       }
}

I don't understand why you are not using index naming in output block? You will encounter problems if you will have more than one index. Add something like that:

output {
    elasticsearch { 
         hosts => ["localhost:9200"] 
         index => "YOUR_INDEX_NAME-%{+YYYY.MM.dd}"
    }
}