检查spark中Dataframes中的匹配数据

问题描述

如果按照我的方式生成的两个 RDD 包含相同的数据（包括行数），我该如何匹配？ 我正在使用 scala 测试来运行测试并使用 scala 2.12.12 启动版本 3 在创建我的两个 rdd 模式的代码下方，包括预期的一个和创建所有 3 个 rdd 数据。

-- CREATING SCHEMA FOR RDD AMOUNTS AND WAREHOUSE AND EXPECTED FINAL SCHEMA

  val amountsSchema: StructType = StructType(
    Seq(
      StructField("positionId", LongType, nullable = true),
      StructField("amount", DecimalType(10, 2), nullable = true),
      StructField("eventTime",LongType, nullable = true),
    )
  )

  val warehouseSchema: StructType = StructType(
    Seq(
      StructField("positionId", LongType, nullable = true),
      StructField("warehouse", StringType, nullable = true),
      StructField("product", StringType, nullable = true),
      StructField("eventTime",LongType, nullable = true),
    )
  )

  val expectedDfSchema: StructType = StructType(
    Seq(
      StructField("positionId", LongType, nullable = true),
      StructField("warehouse", StringType, nullable = true),
      StructField("product", StringType, nullable = true),
      StructField("amount", DecimalType(10, 2), nullable = true),
    )
  )

--- CREATING DATA FOR RDD AMOUNTS RDD AND WAREHOUSE RDD AND EXPECTED FINAL RDD

  val amounts_data = Seq(
    Row("1", "5.00", "1528463387"),
    Row("1", "7.20", "1528463005"),
    Row("2", "5.00", "1528463097"),
    Row("2", "7.20", "1528463007"),
    Row("3", "6.00", "1528463078"),
    Row("4", "24.20", "1528463008"),
    Row("4", "15.00", "1528463100"),
  )


  val wh_data = Seq(
    Row("1", "W-1", "P-1", "1528463098"),
    Row("2", "W-2", "P-2", "1528463097"),
    Row("3", "W-2", "P-3", "1528463078"),
    Row("4", "W-1", "P-6", "1528463100"),
  )

  val expected_data = Seq(
    Row("1", "W-1", "P-1", "5.00"),
    Row("2", "W-2", "P-2", "5.00"),
    Row("3", "W-2", "P-3", "6.00"),
    Row("4", "W-1", "P-6", "15.00")
  )

---- CREATING RDD WITH SCHEMAS AND DATA FOR DF_AMOUNTS AND DF_WAREHOUSE AND FOR THE EXPECTED RDD WITH EXPECTED_DATA

  val df_amounts: DataFrame = spark.createDataFrame(
    spark.sparkContext.parallelize(amounts_data),
    amountsSchema
  )

  val df_wh: DataFrame = spark.createDataFrame(
    spark.sparkContext.parallelize(wh_data),
    warehouseSchema
  )

  val df_expected: DataFrame = spark.createDataFrame(
    spark.sparkContext.parallelize(expected_data),
    expectedDfSchema
  )

---- USING GET_AMOUNTS METHOD TO GENERATE A RDD FROM THE FUNCTION get_amounts



  val resDf: DataFrame = get_amounts(df_amounts, df_wh)



---- TESTING IF THE resDf SCHEMA MATCH WITH THE EXPECTED SCHEMA - IT DOES TEST PASSED

  test("DataFrame Schema Test") {
    assert(assertSchema(resDf.schema, df_expected.schema))
  }

---- TESTING IF THE resDf DATA MATCH WITH THE EXPECTED DATA - IT DOESNT'T MATCH
  test("DataFrame Data Test") {
    assert(assertData(resDf, df_expected))
  }
}

assertData function 用于匹配预期数据 rdd 的数据和来自我的 function get_amounts 的数据，但它未通过测试。

def assertData(df1: DataFrame, df2: DataFrame): Boolean = {
    df1.exceptAll(df2).rdd.isEmpty()
  }

谢谢