Clikhouse + 亚马逊 SNS 通知

Question

I try to insert a Amazon SNS notification eventype = Open to ClickHouse, the Json schema is complex so I don't how I can create my table ( with nested inside a nested ... )

{
  "eventType":"Open",
  "mail":{
    "commonHeaders":{
      "from":[
        "sender@example.com"
      ],
      "messageId":"EXAMPLE7c191be45-e9aedb9a-02f9-4d12-a87d-dd0099a07f8a-000000",
      "subject":"Message sent from Amazon SES",
      "to":[
        "recipient@example.com"
      ]
    },
    "destination":[
      "recipient@example.com"
    ],
    "headers ":[
      {
        "name":"X-SES-CONFIGURATION-SET",
        "value":"ConfigSet"
      },
      {
        "name":"X-SES-MESSAGE-TAGS",
        "value":"myCustomTag1=myCustomValue1, myCustomTag2=myCustomValue2"
      },
      {
        "name":"From",
        "value":"sender@example.com"
      },
      {
        "name":"To",
        "value":"recipient@example.com"
      },
      {
        "name":"Subject",
        "value":"Message sent from Amazon SES"
      },
      {
        "name":"MIME-Version",
        "value":"1.0"
      },
      {
        "name":"Content-Type",
        "value":"multipart/alternative; boundary=\"XBoundary\""
      }
    ],
    "headersTruncated":false,
    "messageId":"EXAMPLE7c191be45-e9aedb9a-02f9-4d12-a87d-dd0099a07f8a-000000",
    "sendingAccountId":"123456789012",
    "source":"sender@example.com",
    "tags":{
      "myCustomTag1":[
        "myCustomValue1"
      ],
      "myCustomTag2":[
        "myCustomValue2"
      ],
      "ses:caller-identity":[
        "ses-user"
      ],
      "ses:configuration-set":[
        "ConfigSet"
      ],
      "ses:from-domain":[
        "example.com"
      ],
      "ses:source-ip":[
        "192.0.2.0"
      ]
    },
    "timestamp":"2017-08-09T21:59:49.927Z"
  },
  "open":{
    "ipAddress":"192.0.2.1",
    "timestamp":"2017-08-09T22:00:19.652Z",
    "userAgent":"Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_3 like Mac OS X) AppleWebKit/603.3.8 (KHTML, like Gecko) Mobile/14G60"
  }
}

I tried INSERT INTO Open FORMAT JSONEachRow and INSERT INTO Open FORMAT JSONCompact but doesn't work.

Thank you.

Answer 1

You should transform your JSON to more simple form without nesting and use JSONEachRow.

Or insert data to CH as JSONAsString and transform using JSONExtract

create table i(J String) Engine=Null;
create table f(a String, i Int64, f Float64) Engine=MergeTree order by a;

create materialized view vv to f
as select (JSONExtract(J, 'Tuple(String,Tuple(Int64,Float64))') as x),
        x.1 as a,
        x.2.1 as i,
        x.2.2 as f
from i;

echo '{"s": "val1", "b2": {"i": 42, "f": 0.1}}' |clickhouse-client -q "insert into i format JSONAsString"

select * from f
┌─a────┬──i─┬───f─┐
│ val1 │ 42 │ 0.1 │
└──────┴────┴─────┘