Java堆空间错误的Pyspark错误

Question

I am new to spark using Spark 1.6.1 with two workers each having Memory 1GB and 5 Cores assigned, running this code on a 33MB file.

This Code is used to Index word in spark.

from textblob import TextBlob as tb
from textblob_aptagger import PerceptronTagger
import numpy as np
import nltk.data
import Constants
from pyspark import SparkContext,SparkConf
import nltk

TOKENIZER = nltk.data.load('tokenizers/punkt/english.pickle')

def word_tokenize(x):
   return nltk.word_tokenize(x)

def pos_tag (s):
  global TAGGER
  return TAGGER.tag(s)

def wrap_words (pair):
  ''' associable each word with index '''
  index = pair[0]
  result = []
  for word, tag in pair[1]:
    word = word.lower()
    result.append({ "index": index, "word": word, "tag": tag})
    index += 1
  return result

if __name__ == '__main__':

  conf = SparkConf().setMaster(Constants.MASTER_URL).setAppName(Constants.APP_NAME)
  sc = SparkContext(conf=conf)
  data = sc.textFile(Constants.FILE_PATH)

  sent = data.flatMap(word_tokenize).map(pos_tag).map(lambda x: x[0]).glom()
  num_partition = sent.getNumPartitions()
  base = list(np.cumsum(np.array(sent.map(len).collect())))
  base.insert(0, 0)
  base.pop()
  RDD = sc.parallelize(base,num_partition)
  tagged_doc = RDD.zip(sent).map(wrap_words).cache()

For Smaller File < 25MB the code is working fine but gives error for files whose size is larger that 25MB.
Help me resolve this issue or provide an alternative to this problem ?

Answer 1

That's because of the .collect(). You lose everything when you transform your rdd into a classic python variable (or np.array), all data is collected on the same place.