在 scala 中使用函数时出现错误类型不匹配

Question

import org.apache.spark.sql.{SparkSession, DataFrame}
import org.apache.spark.sql.functions._

object sparkcpp
{

  val spark = SparkSession.builder().getOrCreate()
  import spark.implicits._
  def extract(): DataFrame =
    {
      val df = spark.read.option("inferschema","true").option("header","true").csv("olive\\A.csv")
     }
 
  def transform(df: DataFrame): DataFrame =
    {

      val df = df.select(df("name"),df("age"))
    }
  def load(df: DataFrame): DataFrame =
    {
      val df = df.write.csv("testing.csv")
    }

  def main(args: Array[String]): Unit =
    {
      load(transform(extract()))
    }
}

*In the above code i'm getting the 'type mismatch error' when I try to load the dataframe in a csv file

*This is the first time i'm working with scala using function. Am I doing it right?

*The aim of the program is to load the dataframe defined to the specified location using function.

• My major concern is the functions that i've used..is it right? please make changes as required.

Answer 1

import org.apache.spark.sql.{SparkSession, DataFrame}
import org.apache.spark.sql.functions._

object sparkcpp
{

  val spark = SparkSession.builder().getOrCreate()
  import spark.implicits._
  def extract(): DataFrame =
    {
       spark.read.option("inferschema","true").option("header","true").csv("olive\\A.csv")
      

     }
 
  def transform(df: DataFrame): DataFrame =
    {

      df.select(df("name"),df("age"))
      
    }
  def load(df: DataFrame): DataFrame =
    {
      df.write.csv("testing.csv")
    }

  def main(args: Array[String]): Unit =
    {
      load(transform(extract()))
    }
}

You should return the dataframe from extract and load method