使用 Scala 展平 JSON

Question

我有一個包含以下數據的 json 文件

{
    "@odata.context": "XXXX",
    "value": [
        {
            "@odata.etag": "W/\"JzQ0OzlxaDNzLys1WXBPbWFXaE5MbFdKbVpNYjMrWDQ1MmJSeGdxVVhrTVRZUXc9MTswMDsn\"",
            "E_No": 345345,
            "G_Code": "007",
            "G_2_Code": ""
        },
        {
            "@odata.etag": "W/\"JzQ0O0ZNWkF2OGd1dVE2L21OQTdKR2g4YU05TldKMERpMUpMWTRSazFKQzZuTDQ9MTswMDsn\"",
            "E_No": 234543,
            "G_Code": "008",
            "G_2_Code": ""
        }
    ],
    "@odata.nextLink": "XXXX"
}

我正在嘗試使用 Scala 在 Databricks 中將其展平。我創建了一個 dataframe DF

val DF= spark.read.json(path)

我想將其作為 json 提供，我需要一個僅使用 E_No、G_Code 和 G_2_Code 創建的 dataframe。 Rest 的列可以從 dataframe 中刪除

我試圖將這個 json 提供到我在其中一個博客中找到的展平代碼中

def flattenDataframe(df: DataFrame): DataFrame = {

    val fields = df.schema.fields
    val fieldNames = fields.map(x => x.name)
    val length = fields.length
    
    for(i <- 0 to fields.length-1){
      val field = fields(i)
      val fieldtype = field.dataType
      val fieldName = field.name
      fieldtype match {
        case arrayType: ArrayType =>
          val fieldNamesExcludingArray = fieldNames.filter(_!=fieldName)
          val fieldNamesAndExplode = fieldNamesExcludingArray ++ Array(s"explode_outer($fieldName) as $fieldName")
          val explodedDf = df.selectExpr(fieldNamesAndExplode:_*)
          return flattenDataframe(explodedDf)
        case structType: StructType =>
          val childFieldnames = structType.fieldNames.map(childname => fieldName +"."+childname)
          val newfieldNames = fieldNames.filter(_!= fieldName) ++ childFieldnames
          val renamedcols = newfieldNames.map(x => (col(x.toString()).as(x.toString().replace(".", "_"))))
         val explodedf = df.select(renamedcols:_*)
          return flattenDataframe(explodedf)
        case _ =>
      }
    }
    df
  }

當我運行以下命令時，出現錯誤

val flattendedJSON = flattenDataframe(DF)

extraneous input '@' expecting {'(', 'COLLECT', 'CONVERT', 'DELTA', 'HISTORY', 'MATCHED', 'MERGE', 'OPTIMIZE', 'SAMPLE', 'TIMESTAMP', 'UPDATE', 'VERSION', 'ZORDER', 'ADD', 'AFTER', 'ALL', 'ALTER', 'ANALYZE', 'AND', 'ANTI', 'ANY', 'ARCHIVE', 'ARRAY', 'AS', 'ASC', 'AT', 'AUTHORIZATION', 'BETWEEN', 'BOTH', 'BUCKET', 'BUCKETS', 'BY', 'CACHE', 'CASCADE', 'CASE', 'CAST', 'CHANGE', 'CHECK', 'CLEAR', 'CLONE', 'CLUSTER', 'CLUSTERED', 'CODEGEN', 'COLLATE', 'COLLECTION', 'COLUMN', 'COLUMNS', 'COMMENT', 'COMMIT', 'COMPACT', 'COMPACTIONS', 'COMPUTE', 'CONCATENATE', 'CONSTRAINT', 'COPY', 'COPY_OPTIONS', 'COST', 'CREATE', 'CREDENTIALS', 'CROSS', 'CUBE', 'CURRENT', 'CURRENT_DATE', 'CURRENT_TIME', 'CURRENT_TIMESTAMP', 'CURRENT_USER', 'DATA', 'DATABASE', DATABASES, 'DAY', 'DBPROPERTIES', 'DEEP', 'DEFINED', 'DELETE', 'DELIMITED', 'DESC', 'DESCRIBE', 'DFS', 'DIRECTORIES', 'DIRECTORY', 'DISTINCT', 'DISTRIBUTE', 'DROP', 'ELSE', 'ENCRYPTION', 'END', 'ESCAPE', 'ESCAPED', 'EXCEPT', 'EXCHANGE', 'EXISTS', 'EXPLAIN', 'EXPORT', 'EXTENDED', 'EXTERNAL', 'EXTRACT', 'FALSE', 'FETCH', 'FIELDS', 'FILTER', 'FILEFORMAT', 'FILES', 'FIRST', 'FOLLOWING', 'FOR', 'FOREIGN', 'FORMAT', 'FORMAT_OPTIONS', 'FORMATTED', 'FROM', 'FULL', 'FUNCTION', 'FUNCTIONS', 'GLOBAL', 'GRANT', 'GROUP', 'GROUPING', 'HAVING', 'HOUR', 'IF', 'IGNORE', 'IMPORT', 'IN', 'INDEX', 'INDEXES', 'INNER', 'INPATH', 'INPUTFORMAT', 'INSERT', 'INTERSECT', 'INTERVAL', 'INTO', 'IS', 'ITEMS', 'JOIN', 'KEYS', 'LAST', 'LATERAL', 'LAZY', 'LEADING', 'LEFT', 'LIKE', 'LIMIT', 'LINES', 'LIST', 'LOAD', 'LOCAL', 'LOCATION', 'LOCK', 'LOCKS', 'LOGICAL', 'MACRO', 'MAP', 'MINUTE', 'MONTH', 'MSCK', 'NAMESPACE', 'NAMESPACES', 'NATURAL', 'NO', NOT, 'NULL', 'NULLS', 'OF', 'ON', 'ONLY', 'OPTION', 'OPTIONS', 'OR', 'ORDER', 'OUT', 'OUTER', 'OUTPUTFORMAT', 'OVER', 'OVERLAPS', 'OVERLAY', 'OVERWRITE', 'PARTITION', 'PARTITIONED', 'PARTITIONS', 'PATTERN', 'PERCENT', 'PIVOT', 'PLACING', 'POSITION', 'PRECEDING', 'PRIMARY', 'PRINCIPALS', 'PROPERTIES', 'PURGE', 'QUERY', 'RANGE', 'RECORDREADER', 'RECORDWRITER', 'RECOVER', 'REDUCE', 'REFERENCES', 'REFRESH', 'RENAME', 'REPAIR', 'REPLACE', 'RESET', 'RESTRICT', 'REVOKE', 'RIGHT', RLIKE, 'ROLE', 'ROLES', 'ROLLBACK', 'ROLLUP', 'ROW', 'ROWS', 'SCHEMA', 'SECOND', 'SELECT', 'SEMI', 'SEPARATED', 'SERDE', 'SERDEPROPERTIES', 'SESSION_USER', 'SET', 'MINUS', 'SETS', 'SHALLOW', 'SHOW', 'SKEWED', 'SOME', 'SORT', 'SORTED', 'START', 'STATISTICS', 'STORED', 'STRATIFY', 'STRUCT', 'SUBSTR', 'SUBSTRING', 'TABLE', 'TABLES', 'TABLESAMPLE', 'TBLPROPERTIES', TEMPORARY, 'TERMINATED', 'THEN', 'TO', 'TOUCH', 'TRAILING', 'TRANSACTION', 'TRANSACTIONS', 'TRANSFORM', 'TRIM', 'TRUE', 'TRUNCATE', 'TYPE', 'UNARCHIVE', 'UNBOUNDED', 'UNCACHE', 'UNION', 'UNIQUE', 'UNKNOWN', 'UNLOCK', 'UNSET', 'USE', 'USER', 'USING', 'VALUES', 'VIEW', 'VIEWS', 'WHEN', 'WHERE', 'WINDOW', 'WITH', 'YEAR', '+', '-', '*', 'DIV', '~', STRING, BIGINT_LITERAL, SMALLINT_LITERAL, TINYINT_LITERAL, INTEGER_VALUE, EXPONENT_VALUE, DECIMAL_VALUE, DOUBLE_LITERAL, BIGDECIMAL_LITERAL, IDENTIFIER, BACKQUOTED_IDENTIFIER}(line 1, pos 0)

== SQL ==
@odata.context
^^^

我猜它以某種方式不喜歡我也不需要的“@odata”列。 我需要去掉那一列，然后看看這種扁平化是否有效。

如果除了我正在使用的扁平化代碼之外還有其他更好的扁平化方法，請告訴我。

謝謝

Answer 1

展開嵌套數組 JSON 和 select 您想要的字段然后以 JSON 格式寫入文件。

val jsonDF= spark.read.json(path)

val explodeColName = "value" // name of the column we want to explode
val flattenColName = explodeColName + "_flat" // temp name

val listOfColsFromArrayType =
  jsonDF.schema
    .find(
      s => s.name == explodeColName && s.dataType.isInstanceOf[ArrayType])
    .map(
      _.dataType
        .asInstanceOf[ArrayType]
        .elementType
        .asInstanceOf[StructType]
        .names
    )

val filterColList =
  listOfColsFromArrayType.getOrElse(throw new Exception("explode Col Name not found")) // or handle the error as needed

val flattenFilterCols = filterColList.map { c =>
  if (c.contains(".")) { col(s"$flattenColName.`$c`") } else {
    col(s"$flattenColName.$c")
  }
}

val flatten = jsonDF
  .select(explode(col(explodeColName)).as(flattenColName))
  .select(flattenFilterCols: _*)
   
    flattenDF
      .write
      .json(outputPath)

結果將是

{"@odata.etag":"W/\"JzQ0OzlxaDNzLys1WXBPbWFXaE5MbFdKbVpNYjMrWDQ1MmJSeGdxVVhrTVRZUXc9MTswMDsn\"","E_No":345345,"G_2_Code":"","G_Code":"007"}
{"@odata.etag":"W/\"JzQ0O0ZNWkF2OGd1dVE2L21OQTdKR2g4YU05TldKMERpMUpMWTRSazFKQzZuTDQ9MTswMDsn\"","E_No":234543,"G_2_Code":"","G_Code":"008"}

Answer 2

我對你的方法做了一些改動，現在它正在工作。

請注意，我沒有重命名任何基礎列。 如果您想在進一步處理中獲取它，請使用 backtique (`)

測試數據

 DF.show(false)
    DF.printSchema()

    /**
      * +--------------+---------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
      * |@odata.context|@odata.nextLink|value                                                                                                                                                                                         |
      * +--------------+---------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
      * |XXXX          |XXXX           |[[W/"JzQ0OzlxaDNzLys1WXBPbWFXaE5MbFdKbVpNYjMrWDQ1MmJSeGdxVVhrTVRZUXc9MTswMDsn", 345345, , 007], [W/"JzQ0O0ZNWkF2OGd1dVE2L21OQTdKR2g4YU05TldKMERpMUpMWTRSazFKQzZuTDQ9MTswMDsn", 234543, , 008]]|
      * +--------------+---------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
      *
      * root
      * |-- @odata.context: string (nullable = true)
      * |-- @odata.nextLink: string (nullable = true)
      * |-- value: array (nullable = true)
      * |    |-- element: struct (containsNull = true)
      * |    |    |-- @odata.etag: string (nullable = true)
      * |    |    |-- E_No: long (nullable = true)
      * |    |    |-- G_2_Code: string (nullable = true)
      * |    |    |-- G_Code: string (nullable = true)
      *
      */

展平數組和結構類型的嵌套列

    def flattenDataframe(df: DataFrame): DataFrame = {

      val fields = df.schema.fields
      val fieldNames = fields.map(x => x.name)
      val length = fields.length

      for(i <- 0 to fields.length-1){
        val field = fields(i)
        val fieldtype = field.dataType
        val fieldName = field.name
        fieldtype match {
          case arrayType: ArrayType =>
            val fieldNamesExcludingArray = fieldNames.filter(_!=fieldName)
            val fieldNamesAndExplode = fieldNamesExcludingArray.map(c => s"`$c`") ++
              Array(s"explode_outer($fieldName) as $fieldName")
            val explodedDf = df.selectExpr(fieldNamesAndExplode:_*)
            return flattenDataframe(explodedDf)
          case structType: StructType =>
            val childFieldnames = structType.fieldNames.map(childname => s"$fieldName.`$childname`")
            val newfieldNames = fieldNames.filter(_!= fieldName).map(c => s"`$c`") ++ childFieldnames
            val renamedcols = newfieldNames.map(x => col(x))
            val explodedf = df.select(renamedcols:_*)
            return flattenDataframe(explodedf)
          case _ =>
        }
      }
      df
    }

    val flattendedJSON = flattenDataframe(DF)
    flattendedJSON.show(false)
    flattendedJSON.printSchema()

    /**
      * +--------------+---------------+----------------------------------------------------------------------------+------+--------+------+
      * |@odata.context|@odata.nextLink|@odata.etag                                                                 |E_No  |G_2_Code|G_Code|
      * +--------------+---------------+----------------------------------------------------------------------------+------+--------+------+
      * |XXXX          |XXXX           |W/"JzQ0OzlxaDNzLys1WXBPbWFXaE5MbFdKbVpNYjMrWDQ1MmJSeGdxVVhrTVRZUXc9MTswMDsn"|345345|        |007   |
      * |XXXX          |XXXX           |W/"JzQ0O0ZNWkF2OGd1dVE2L21OQTdKR2g4YU05TldKMERpMUpMWTRSazFKQzZuTDQ9MTswMDsn"|234543|        |008   |
      * +--------------+---------------+----------------------------------------------------------------------------+------+--------+------+
      *
      * root
      * |-- @odata.context: string (nullable = true)
      * |-- @odata.nextLink: string (nullable = true)
      * |-- @odata.etag: string (nullable = true)
      * |-- E_No: long (nullable = true)
      * |-- G_2_Code: string (nullable = true)
      * |-- G_Code: string (nullable = true)
      */