用fluentd和winston解析json

Question

My application generates apche logs as well as JSON data something like this

{ TableName: 'myTable', CapacityUnits: 0.5 }

I am using winston(3.2.1) as my logger. In my Kibana, I see each line of JSON as a different entry instead of a single json. Any idea how to solve this?

My winston code looks like this

const winston = require('winston');

const { format } = winston;

const prettyJson = format.printf((info) => {
  if (info.message.constructor === Object) {
    info.message = JSON.stringify(info.message, null, 2);
    console.log('inside pretyjson', info.message);
  }
  return `${info.level}: ${info.message}`;
});

const logLevel = process.env.LOG_LEVEL || 'debug';

const tsFormat = () => (new Date()).toLocaleTimeString();

const Logger = winston.createLogger({
  level: logLevel,
  transports: [
    new winston.transports.Console({
      timestamp: tsFormat,
      format: format.combine(
        format.colorize(),
        format.prettyPrint(),
        format.splat(),
        format.simple(),
        prettyJson,
      ),
    }),
  ],
});

module.exports = Logger;

My fluentd config looks like this

# Recieve events over http from port 9880
<source>
  @type http
  port 9880
  bind 0.0.0.0
  @log_level debug
</source>
# Recieve events from 24224/tcp
<source>
  @type forward
  port 24224
  bind 0.0.0.0
</source>
# We need to massage the data before if goes into the ES
<filter **>
  # We parse the input with key "log" (https://docs.fluentd.org/filter/parser)
  @type parser
  key_name log
  # Keep the original key value pair in the result
  reserve_data true
  <parse>
    # Use apache2 parser plugin to parse the data
    @type multi_format
   <pattern>
      format json
    </pattern>
    <pattern>
      format apache2
    </pattern>
     <pattern>
      format none
    </pattern>
</parse>
</filter>
# Fluentd will decide what to do here if the event is matched
# In our case, we want all the data to be matched hence **
<match **>
# We want all the data to be copied to elasticsearch using inbuilt
# copy output plugin https://docs.fluentd.org/output/copy
  @type copy
  <store>
  # We want to store our data to elastic search using out_elasticsearch plugin
  # https://docs.fluentd.org/output/elasticsearch. See Dockerfile for installation
    @type elasticsearch
    time_key timestamp_ms
    host hostip
    port 9200
    with_transporter_log true
    @log_level debug
    log_es_400_reason true
    # Use conventional index name format (logstash-%Y.%m.%d)
    logstash_format true
    # We will use this when kibana reads logs from ES
    logstash_prefix fluentd
    logstash_dateformat %Y-%m-%d
    flush_interval 1s
    reload_connections false
    reconnect_on_error true
    reload_on_failure true
  </store>
</match>

Answer 1

maybe it's too late, but if you use:

format: format.combine(        
    format.prettyPrint(),        
    prettyJson
  ),

That generates a code with each line of JSON to stdout. Fluentd read that like different lines. You can delete those lines or use something like fluentd multiline.

https://docs.fluentd.org/parser/multiline