Grok patterns to match log with multiple special characters

Question

I want to catch my exception with ELK but my exception is full of ( { [ . , \\ / , " ' character. How can I index them in grok?

My log file:

Exception in *** CoreLevel*** occured. 
Date&Time: 2018-01-21 09:52:20.744092 
Root: 
 ( ['MQROOT' : 0x7f0a902b2d80]
  (0x01000000:Name  ):Properties = ( ['MQPROPERTYPARSER' : 0x7f0a902bffa0]
    (0x03000000:NameValue):MessageFormat          = 'jms_text' (CHARACTER)  )
    (0x03000000:NameValue):MsgId            = X'5059414d313339363131303234383030303238' (BLOB))
    (0x01000000:Name     ):usr            = (
      (0x03000000:NameValue):MessageName = 'SampleMessageName' (CHARACTER)
      (0x03000000:NameValue):MsgVersion  = 'V1' (CHARACTER)
    )
  )
) 
***************************************************************************************** 
***************************************************************************************** 
ExceptionList:  
( ['MQROOT' : 0x7f0a9072b350]
  (0x01000000:Name):RecoverableException = (
    (0x03000000:NameValue):File                 = '/build/slot1/S800_P/src/DataFlowEngine/PluginInterface/ImbJniNode.cpp' (CHARACTER)
    (0x03000000:NameValue):Line                 = 1260 (INTEGER)
    (0x03000000:NameValue):Text                 = 'Caught exception and rethrowing' (CHARACTER)
    (0x01000000:Name     ):Insert               = (
      (0x03000000:NameValue):Type = 14 (INTEGER)
    )
          (0x03000000:NameValue):Label    = '' (CHARACTER)
          (0x03000000:NameValue):Catalog  = "BIPmsgs" (CHARACTER)
          (0x03000000:NameValue):Severity = 3 (INTEGER)
          (0x03000000:NameValue):Number   = 4395 (INTEGER)
   )
  )
)

and I except to get this pattern into kibana

Exception in: CoreLevel, 
Date&Time: 2018-01-21 09:52:20.744092
message:{
  Root:".....",
  ExceptionList:"......"
}

and this is my grok block that doesn't work

 grok {
    patterns_dir => "/etc/logstash/patterns/"
    break_on_match => false
    keep_empty_captures => true

    match => {"message" => ["Exception in (?<msg_f> occured..) Date&Time: %{SYSLOGTIMESTAMP:timestamp}"]}
 }

  mutate {
        gsub => ["message", "\n", ""]
 }

I'd really appreciate if anyone could help me.

Answer 1

The date in your log is in ISO8601 format so it can be matched with TIMESTAMP_ISO8601 predefined pattern. For lines after date & time, you can use (?m) to match multiline in your log with GREEDYDATA .

Following pattern will work,

Exception in \*\*\* %{WORD:Exception_in}.*\s*Date&Time: %{TIMESTAMP_ISO8601}(?m)%{GREEDYDATA}

It will output,

{
  "Exception_in": [
    [
      "CoreLevel"
    ]
  ],
  "TIMESTAMP_ISO8601": [
    [
      "2018-01-21 09:52:20.744092"
    ]
  ],
  "YEAR": [
    [
      "2018"
    ]
  ],
  "MONTHNUM": [
    [
      "01"
    ]
  ],
  "MONTHDAY": [
    [
      "21"
    ]
  ],
  "HOUR": [
    [
      "09",
      null
    ]
  ],
  "MINUTE": [
    [
      "52",
      null
    ]
  ],
  "SECOND": [
    [
      "20.744092"
    ]
  ],
  "ISO8601_TIMEZONE": [
    [
      null
    ]
  ],
  "GREEDYDATA": [
    [
      " \nRoot: \n ( ['MQROOT' : 0x7f0a902b2d80]\n  (0x01000000:Name  ):Properties = ( ['MQPROPERTYPARSER' : 0x7f0a902bffa0]\n    (0x03000000:NameValue):MessageFormat          = 'jms_text' (CHARACTER)  )\n    (0x03000000:NameValue):MsgId            = X'5059414d313339363131303234383030303238' (BLOB))\n    (0x01000000:Name     ):usr            = (\n      (0x03000000:NameValue):MessageName = 'SampleMessageName' (CHARACTER)\n      (0x03000000:NameValue):MsgVersion  = 'V1' (CHARACTER)\n    )\n  )\n) \n***************************************************************************************** \n***************************************************************************************** \nExceptionList:  \n( ['MQROOT' : 0x7f0a9072b350]\n  (0x01000000:Name):RecoverableException = (\n    (0x03000000:NameValue):File                 = '/build/slot1/S800_P/src/DataFlowEngine/PluginInterface/ImbJniNode.cpp' (CHARACTER)\n    (0x03000000:NameValue):Line                 = 1260 (INTEGER)\n    (0x03000000:NameValue):Text                 = 'Caught exception and rethrowing' (CHARACTER)\n    (0x01000000:Name     ):Insert               = (\n      (0x03000000:NameValue):Type = 14 (INTEGER)\n    )\n          (0x03000000:NameValue):Label    = '' (CHARACTER)\n          (0x03000000:NameValue):Catalog  = "BIPmsgs" (CHARACTER)\n          (0x03000000:NameValue):Severity = 3 (INTEGER)\n          (0x03000000:NameValue):Number   = 4395 (INTEGER)\n   )\n  )\n)"
    ]
  ]
}

You can test it here