如何在Spark Streaming中使用無限的Scala流作為源？

Question

假設我基本上希望Stream.from(0)作為InputDStream 。 我該怎么做？ 我能看到的唯一方法是使用StreamingContext#queueStream ，但是我必須從另一個線程或子類Queue排隊元素以創建一個行為類似於無限流的隊列，這兩者都感覺像是一個黑客。

這樣做的正確方法是什么？

Answer 1

我不認為默認情況下它在Spark中可用，但使用ReceiverInputDStream很容易實現它。

import org.apache.spark.storage.StorageLevel
import org.apache.spark.streaming.StreamingContext
import org.apache.spark.streaming.dstream.ReceiverInputDStream
import org.apache.spark.streaming.receiver.Receiver

class InfiniteStreamInputDStream[T](
       @transient ssc_ : StreamingContext,
       stream: Stream[T],
       storageLevel: StorageLevel
      ) extends ReceiverInputDStream[T](ssc_)  {

  override def getReceiver(): Receiver[T] = {
    new InfiniteStreamReceiver(stream, storageLevel)
  }
}

class InfiniteStreamReceiver[T](stream: Stream[T], storageLevel: StorageLevel) extends Receiver[T](storageLevel) {

  // Stateful iterator
  private val streamIterator = stream.iterator

  private class ReadAndStore extends Runnable {
    def run(): Unit = {
      while (streamIterator.hasNext) {
        val next = streamIterator.next()
        store(next)
      }
    }
  }

  override def onStart(): Unit = {
    new Thread(new ReadAndStore).run()    
  }

  override def onStop(): Unit = { }
}

Answer 2

稍微修改過的代碼與Spark 2.0一起使用：

import org.apache.spark.storage.StorageLevel
import org.apache.spark.streaming.StreamingContext
import org.apache.spark.streaming.dstream.ReceiverInputDStream
import org.apache.spark.streaming.receiver.Receiver
import scala.reflect.ClassTag

class InfiniteDStream[T: ClassTag](
                                    @transient ssc_ : StreamingContext,
                                    stream: Stream[T],
                                    storageLevel: StorageLevel
                                  ) extends ReceiverInputDStream[T](ssc_) {

  override def getReceiver(): Receiver[T] = {
    new InfiniteStreamReceiver(stream, storageLevel)
  }
}

class InfiniteStreamReceiver[T](stream: Stream[T], storageLevel: StorageLevel) extends Receiver[T](storageLevel) {

  private class ReadAndStore extends Runnable {
    def run(): Unit = {
      stream.foreach(store)
    }
  }

  override def onStart(): Unit = {
    val t = new Thread(new ReadAndStore)
    t.setDaemon(true)
    t.run()
  }

  override def onStop(): Unit = {}
}