Multilevel Complex Nested Json Using Spark SQL

Question

I have a specific requirement to convert some related tables data in nested json like below by using Spark SQL. I have achieved it with Scala but not getting it resolved in Spark SQL.

{
"REPORTING_CARRIER":"9E", "DISTANCE":"3132", "ORIGIN_STATE_NM":"Pennsylvania", "QUARTER":"2", "YEAR":"2017", "ITIN_GEO_TYPE":"2", "BULK_FARE":"0", "ORIGIN":"ABE", "ORIGIN_AIRPORT_ID":"10135", "ITIN_FARE":"787", "ORIGIN_CITY_MARKET_ID":"30135", "ROUNDTRIP":"1", "Market":[
{
"MKT_DISTANCE":"1566", "MKT_BULK_FARE":"0", "MKT_NO_OF_CPNS":"2", "MKT_DEST_STATE_NM":"Texas", "MKT_OP_CARR_GRP":"9E:DL", "MKT_TK_CARR_GRP":"DL:DL", "MKT_MILES_FLOWN":"1566", "MKT_AIRPORT_GROUP":"ABE:ATL:SAT", "MKT_FARE_AMT":"393.5", "MKT_ORIG_STATE_NM":"Pennsylvania", "MKT_DEST_ARPT_CITY_NM":"33214", "MKT_RPTG_CARR_NM":"9E", "MKT_DEST":"SAT", "MKT_DEST_CNTRY":"US", "MKT_ORIG_CNTRY":"US", "Coupon":[
{
"CPN_STATE_NM":"Georgia", "CPN_DEST":"ATL", "CPN_TKT_CARR_NM":"DL", "TRIP_BREAK":"", "CPN_MKT_ORIG_ARPT_NM":"10135", "CLASS_OF_SVC":"X", "CPN_TKT_NBR":"2017245", "CPN_DEST_CITY_MKT_NM":"30397", "CPN_DISTANCE":"692", "SEQ_NUM":"1", "ITIN_GEO_TYPE":"2", "CPN_RPTG_CARR_NM":"9E", "COUPON_GEO_TYPE":"2", "CPN_ORIG_STATE_NM":"Pennsylvania", "CPN_OPERG_CARR_NM":"9E", "CPN_ORIG":"ABE", "CPN_PASSENGERS":"1", "COUPON_TYPE":"A", "CPN_DEST_ARPT_NM":"10397", "CPN_MKT_ORIG_CITY_NM":"30135", "CPN_DEST_CNTRY":"US", "CPN_MKT_ID":"201724501", "CPN_ORIG_CNTRY":"US" }, {
"CPN_STATE_NM":"Texas", "CPN_DEST":"SAT", "CPN_TKT_CARR_NM":"DL", "TRIP_BREAK":"X", "CPN_MKT_ORIG_ARPT_NM":"10397", "CLASS_OF_SVC":"X", "CPN_TKT_NBR":"2017245", "CPN_DEST_CITY_MKT_NM":"33214", "CPN_DISTANCE":"874", "SEQ_NUM":"2", "ITIN_GEO_TYPE":"2", "CPN_RPTG_CARR_NM":"9E", "COUPON_GEO_TYPE":"2", "CPN_ORIG_STATE_NM":"Georgia", "CPN_OPERG_CARR_NM":"DL", "CPN_ORIG":"ATL", "CPN_PASSENGERS":"1", "COUPON_TYPE":"A", "CPN_DEST_ARPT_NM":"14683", "CPN_MKT_ORIG_CITY_NM":"30397", "CPN_DEST_CNTRY":"US", "CPN_MKT_ID":"201724501", "CPN_ORIG_CNTRY":"US" } ], "MKT_ITIN_ID":"2017245", "MKT_OPERG_CARR_NM":"99", "MKT_DEST_ARPT_NM":"14683", "MKT_ORIG_ARPT_NM":"ABE", "MKT_ITIN_GEO_TYPE":"2", "MKT_PASSENGERS":"1", "MKT_ID":"201724501", "MKT_TKT_CARR_NM":"DL" }, {
"MKT_DISTANCE":"1566", "MKT_BULK_FARE":"0", "MKT_NO_OF_CPNS":"2", "MKT_DEST_STATE_NM":"Pennsylvania", "MKT_OP_CARR_GRP":"DL:DL", "MKT_TK_CARR_GRP":"DL:DL", "MKT_MILES_FLOWN":"1566", "MKT_AIRPORT_GROUP":"SAT:ATL:ABE", "MKT_FARE_AMT":"393.5", "MKT_ORIG_STATE_NM":"Texas", "MKT_DEST_ARPT_CITY_NM":"30135", "MKT_RPTG_CARR_NM":"9E", "MKT_DEST":"ABE", "MKT_DEST_CNTRY":"US", "MKT_ORIG_CNTRY":"US", "Coupon":[
{
"CPN_STATE_NM":"Georgia", "CPN_DEST":"ATL", "CPN_TKT_CARR_NM":"DL", "TRIP_BREAK":"", "CPN_MKT_ORIG_ARPT_NM":"14683", "CLASS_OF_SVC":"X", "CPN_TKT_NBR":"2017245", "CPN_DEST_CITY_MKT_NM":"30397", "CPN_DISTANCE":"874", "SEQ_NUM":"3", "ITIN_GEO_TYPE":"2", "CPN_RPTG_CARR_NM":"9E", "COUPON_GEO_TYPE":"2", "CPN_ORIG_STATE_NM":"Texas", "CPN_OPERG_CARR_NM":"DL", "CPN_ORIG":"SAT", "CPN_PASSENGERS":"1", "COUPON_TYPE":"A", "CPN_DEST_ARPT_NM":"10397", "CPN_MKT_ORIG_CITY_NM":"33214", "CPN_DEST_CNTRY":"US", "CPN_MKT_ID":"201724503", "CPN_ORIG_CNTRY":"US" }, {
"CPN_STATE_NM":"Pennsylvania", "CPN_DEST":"ABE", "CPN_TKT_CARR_NM":"DL", "TRIP_BREAK":"X", "CPN_MKT_ORIG_ARPT_NM":"10397", "CLASS_OF_SVC":"X", "CPN_TKT_NBR":"2017245", "CPN_DEST_CITY_MKT_NM":"30135", "CPN_DISTANCE":"692", "SEQ_NUM":"4", "ITIN_GEO_TYPE":"2", "CPN_RPTG_CARR_NM":"9E", "COUPON_GEO_TYPE":"2", "CPN_ORIG_STATE_NM":"Georgia", "CPN_OPERG_CARR_NM":"DL", "CPN_ORIG":"ATL", "CPN_PASSENGERS":"1", "COUPON_TYPE":"A", "CPN_DEST_ARPT_NM":"10135", "CPN_MKT_ORIG_CITY_NM":"30397", "CPN_DEST_CNTRY":"US", "CPN_MKT_ID":"201724503", "CPN_ORIG_CNTRY":"US" } ], "MKT_ITIN_ID":"2017245", "MKT_OPERG_CARR_NM":"DL", "MKT_DEST_ARPT_NM":"10135", "MKT_ORIG_ARPT_NM":"SAT", "MKT_ITIN_GEO_TYPE":"2", "MKT_PASSENGERS":"1", "MKT_ID":"201724503", "MKT_TKT_CARR_NM":"DL" } ], "NO_OF_CPNS":"4", "ORIGIN_COUNTRY":"US", "ITIN_ID":"2017245", "PASSENGERS":"1", "MILES_FLOWN":"3132" }

Answer 1

You can use the from_json() helper function within the select() Dataset API call, to extract or decode data's attributes and values from a JSON string into a DataFrame as columns, dictated by a schema.

example, given the following json { "reporting_carrier": "A", "market": { "value": 10 } } , stored in the rawJsonDf

case class MarketData (reporting_carrier: String, market_json: String)

val jsonSchema = new StructType()
  .add("value", LongType)

rawJsonDf
  .toDf("reporting_carrier","market")
  .as[MarketData]

rawJsonDf
  .select(from_json($"market_json", jsonSchema) as "market")
  .filter($"market.value" > 5)

see this great tutorial by databricks for more info .