線程“main”中的異常java.lang.NoSuchMethodError：org.apache.spark.sql.SQLContext.sql（Ljava / lang / String;）Lorg / apache / spark / sql / Dataset;

Question

我正在嘗試使用Scala IDE中的Spark SQL，我在沒有Maven的情況下進行設置。 我在生產環境中使用Spark 1.5.1並嘗試執行以下代碼
spark-submit --class com.dataflair.spark.Wordcount --master cluster /home/appserver/Sparkjob.jar

package com.dataflair.spark
import org.apache.spark.SparkContext
import org.apache.spark.SparkConf
import org.apache.spark.SparkContext

import org.apache.spark.SparkContext._

import org.apache.spark.sql.SQLContext

import org.slf4j.Logger
import org.slf4j.LoggerFactory;

object Wordcount {
def main(args: Array[String]) {
//Create conf object
val conf = new SparkConf()
.setAppName("WordCount")
//create spark context object
val sc = new SparkContext(conf)
//Check whether sufficient params are supplied
val sqlcontext=new org.apache.spark.sql.SQLContext(sc)

//Check whether sufficient params are supplied

import sqlcontext.implicits._




val dp_new=sqlcontext.sql("  SELECT * FROM  Temp_UFDR_t1 ")

dp_new.registerTempTable("DFUO")


//stop the spark context
sc.stop
}
}

我有以下問題

Exception in thread "main" java.lang.NoSuchMethodError: org.apache.spark.sql.SQLContext.sql(Ljava/lang/String;)Lorg/apache/spark/sql/Dataset;
        at com.dataflair.spark.Wordcount$.main(Wordcount.scala:26)
        at com.dataflair.spark.Wordcount.main(Wordcount.scala)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:766)
        at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:183)
        at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:208)
        at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:123)
        at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)

Answer 1

正如你所說，你使用的是spark 1.5.1，但我們在spark 1.5.1中沒有DataSet。

可能你已經在Spark 1.5.1中編寫了代碼，並在Spark1.6及Spark Context對象中部署/運行了應用程序。