Apache ignite and spark itegration error java.lang.NoSuchMethodError: org.apache.spark.sql.SQLContext.createDataFrame

Question

We are tring to integrate apache ignite and apache spark, we are getting below error.

Maven config

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>com.me</groupId>
    <artifactId>igniteclient1</artifactId>
    <version>0.0.1-SNAPSHOT</version>
    <packaging>jar</packaging>

    <name>igniteclient1</name>
    <url>http://maven.apache.org</url>

    <properties>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
    </properties>

    <dependencies>
        <!-- https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.10 -->
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-core_2.10</artifactId>
            <version>2.1.0</version>
        </dependency>
        <!-- https://mvnrepository.com/artifact/org.apache.spark/spark-sql_2.10 -->
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-sql_2.10</artifactId>
            <version>2.1.0</version>
        </dependency>


        <!-- https://mvnrepository.com/artifact/org.apache.ignite/ignite-core -->
        <dependency>
            <groupId>org.apache.ignite</groupId>
            <artifactId>ignite-core</artifactId>
            <version>1.8.0</version>
        </dependency>

        <!-- https://mvnrepository.com/artifact/org.apache.ignite/ignite-spring -->
        <dependency>
            <groupId>org.apache.ignite</groupId>
            <artifactId>ignite-spring</artifactId>
            <version>1.8.0</version>
        </dependency>

        <!-- https://mvnrepository.com/artifact/org.apache.ignite/ignite-indexing -->
        <dependency>
            <groupId>org.apache.ignite</groupId>
            <artifactId>ignite-indexing</artifactId>
            <version>1.8.0</version>
        </dependency>

        <!-- https://mvnrepository.com/artifact/org.apache.ignite/ignite-rest-http -->
        <dependency>
            <groupId>org.apache.ignite</groupId>
            <artifactId>ignite-rest-http</artifactId>
            <version>1.8.0</version>
        </dependency>

        <!-- https://mvnrepository.com/artifact/org.apache.ignite/ignite-spark -->
        <dependency>
            <groupId>org.apache.ignite</groupId>
            <artifactId>ignite-spark</artifactId>
            <version>1.8.0</version>
        </dependency>
        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>3.8.1</version>
            <scope>test</scope>
        </dependency>
    </dependencies>
    <build>
        <plugins>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-shade-plugin</artifactId>
                <version>3.0.0</version>
                <configuration>
                    <filters>
                        <filter>
                            <artifact>*:*</artifact>
                            <excludes>
                                <exclude>META-INF/*.SF</exclude>
                                <exclude>META-INF/*.DSA</exclude>
                                <exclude>META-INF/*.RSA</exclude>
                            </excludes>
                        </filter>
                    </filters>
                    <!-- Additional configuration. -->
                </configuration>
                <executions>
                    <execution>
                        <phase>package</phase>
                        <goals>
                            <goal>shade</goal>
                        </goals>
                    </execution>
                </executions>
            </plugin>
        </plugins>
    </build>
</project>

Code:

package com.me.igniteclient1;

import org.apache.spark.api.java.*;
import org.apache.spark.sql.DataFrame;
import org.apache.spark.sql.Row;

import com.me.igniteclient1.ratio.Ratio;

import org.apache.ignite.Ignite;
import org.apache.ignite.Ignition;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.lang.IgniteOutClosure;
import org.apache.ignite.spark.JavaIgniteContext;
import org.apache.ignite.spark.JavaIgniteRDD;
import org.apache.spark.SparkConf;
import org.apache.spark.sql.SQLContext;


/**
 *
 */
public class App 
{
    public static void main(String[] args) {

        SparkConf conf = new SparkConf().setAppName("Stockrank styles");
        JavaSparkContext sc = new JavaSparkContext(conf);

        JavaIgniteContext igniteContext = new JavaIgniteContext(sc, new IgniteConfigProvider());
        JavaIgniteRDD<String, Ratio> javaRdd = igniteContext.fromCache("RatioCache");

        DataFrame df = javaRdd.sql("select ric, ratio1, ratio2, ratio3, ratio4 from Ratio where date = ? and ratios__Exchange = ? and ratio5 > ?", "2017-01-25", "LSE", 10);

        df.printSchema();
        df.show();

        sc.stop();
      }

    /**
     * Ignite configiration provider.
     */
    static class IgniteConfigProvider implements IgniteOutClosure<IgniteConfiguration> {

        public IgniteConfiguration apply() {

            Ignition.setClientMode(true);

            // Start Ignite in client mode.
            Ignite ignite = Ignition.start();
            return ignite.configuration();
        }

    }

}

Error:

 Exception in thread "main" java.lang.NoSuchMethodError: org.apache.spark.sql.SQLContext.createDataFrame(Lorg/apache/spark/rdd/RDD;Lorg/apache/spark/sql/types/StructType;)Lorg/apache/spark/sql/DataFrame;
        at org.apache.ignite.spark.IgniteRDD.sql(IgniteRDD.scala:152)
        at org.apache.ignite.spark.JavaIgniteRDD.sql(JavaIgniteRDD.scala:79)
        at org.apache.ignite.spark.JavaIgniteRDD.sql(JavaIgniteRDD.scala:79)
        at com.me.igniteclient1.App.main(App.java:38)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:738)
        at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:187)
        at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:212)
        at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:126)
        at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
    17/02/10 17:55:10 INFO GridUpdateNotifier: Your version is up to date.

Do these tests pass in ignite spark module ?

https://github.com/apache/ignite/blob/master/modules/spark/src/test/java/org/apache/ignite/spark/JavaStandaloneIgniteRDDSelfTest.java

https://github.com/apache/ignite/blob/master/modules/spark/src/test/java/org/apache/ignite/spark/JavaEmbeddedIgniteRDDSelfTest.java

I tried different spark versions, 1.6, 1.5, 1.4 but nothing worked. I am getting same error.

We are evaluating possibility to use ignite (as inmemory datafabric) and spark (analytics using mlib). But integration module does not seem to be very active. What are the future plans of ? Is it going to be deprecated ?

UPDATE: Ignite 1.9 released https://blogs.apache.org/ignite/entry/apache-ignite-1-9-released

Spark

Ignite's spark integration was upgraded to the latest Spark version. Presently, you can leverage from Ignite Shared RDDs in applications using latest Spark version.

Answer 1

According to POM, you used Spark 2.1.0. Ignite is based on 1.5.2 and currently is not supporting 2.x.

Also ignite-spark module is created for Scala 2.11. If you have 2.10, use ignite-spark_2.10 instead.

Overall, it's definitely a classpath issue. Make sure that all versions are consistent and it should work fine.