Hadoop-MapReduce

Question

我一直在嘗試解決一個簡單的Map / Reduce問題，在該問題中，我將對一些輸入文件中的單詞進行計數，然后將其頻率作為一個鍵，並將其單詞長度作為另一個鍵。 映射將發出一個eveytime，從文件中讀取一個新單詞，然后將所有相同的單詞歸為一組，以得到其最終計數。 然后，作為輸出，我想查看每個單詞長度的統計信息，即最常用的單詞。

這是我們（和我的團隊）所獲得的：這是WordCountMapper類

import java.io.IOException;
import java.util.ArrayList;
import java.util.StringTokenizer;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.MapReduceBase;
import org.apache.hadoop.mapred.Mapper;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reporter;


public class WordCountMapper extends MapReduceBase implements
                Mapper<LongWritable, Text, Text, CompositeGroupKey> {

        private final IntWritable one = new IntWritable(1);
        private Text word = new Text();

                 public void map(LongWritable key, Text value,
                 OutputCollector<Text, CompositeGroupKey> output, Reporter reporter)
                 throws IOException {

                 String line = value.toString();
                 StringTokenizer itr = new StringTokenizer(line.toLowerCase());
                 while(itr.hasMoreTokens()) {
                 word.set(itr.nextToken());
                 CompositeGroupKey gky = new CompositeGroupKey(1, word.getLength());
                 output.collect(word, gky);
                 }
                 }
}

這是wordcountreducer類別：

import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.MapReduceBase;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reducer;
import org.apache.hadoop.mapred.Reporter;

import com.sun.xml.internal.bind.CycleRecoverable.Context;

public class WordCountReducer extends MapReduceBase
    implements Reducer<Text, CompositeGroupKey, Text, CompositeGroupKey> {

        @Override
        public void reduce(Text key, Iterator<CompositeGroupKey> values,
                        OutputCollector<Text, CompositeGroupKey> output, Reporter reporter)
                        throws IOException {
                int sum = 0;
                int length = 0;
                while (values.hasNext()) {
                CompositeGroupKey value = (CompositeGroupKey) values.next();
                sum += (Integer) value.getCount(); // process value
                length = (Integer) key.getLength();
            }
            CompositeGroupKey cgk = new CompositeGroupKey(sum,length);
            output.collect(key, cgk);
        }
}

這是班上的字數統計

import java.util.ArrayList;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.FileInputFormat;
import org.apache.hadoop.mapred.FileOutputFormat;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.JobStatus;
import org.apache.hadoop.mapred.jobcontrol.Job;

import org.apache.hadoop.util.GenericOptionsParser;
import org.apache.hadoop.util.StringUtils;

public class WordCount {

  public static void main(String[] args) {
    JobClient client = new JobClient();
    JobConf conf = new JobConf(WordCount.class);

// specify output types
    conf.setOutputKeyClass(Text.class);
    conf.setOutputValueClass(CompositeGroupKey.class);

    conf.setMapOutputKeyClass(Text.class);
    conf.setMapOutputValueClass(CompositeGroupKey.class);

    // specify input and output dirs
    FileInputFormat.addInputPath(conf, new Path("input"));
    FileOutputFormat.setOutputPath(conf, new Path("output16"));

    // specify a mapper
    conf.setMapperClass(WordCountMapper.class);

// specify a reducer
    conf.setReducerClass(WordCountReducer.class);
    conf.setCombinerClass(WordCountReducer.class);

    client.setConf(conf);
    try {
      JobClient.runJob(conf);
    } catch (Exception e) {
      e.printStackTrace();
    }

  }
}  

And this is the groupcompositekey

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.io.WritableUtils;

public class CompositeGroupKey implements WritableComparable<CompositeGroupKey> {
    int count;
    int length;

    public CompositeGroupKey(int c, int l) {
        this.count = c;
        this.length = l;
    }

    public void write(DataOutput out) throws IOException {
        WritableUtils.writeVInt(out, count);
        WritableUtils.writeVInt(out, length);
    }

    public void readFields(DataInput in) throws IOException {
        this.count = WritableUtils.readVInt(in);
        this.length = WritableUtils.readVInt(in);
    }

    public int compareTo(CompositeGroupKey pop) {
        return 0;
    }

    public int getCount() {
        return this.count;
    }

    public int getLength() {
        return this.length;
    }

}

現在我得到這個錯誤：

 java.lang.RuntimeException: java.lang.NoSuchMethodException: CompositeGroupKey.<init>() at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:80) at org.apache.hadoop.io.serializer.WritableSerialization$WritableDeserializer.deserialize(WritableSerialization.java:62) at org.apache.hadoop.io.serializer.WritableSerialization$WritableDeserializer.deserialize(WritableSerialization.java:40) at org.apache.hadoop.mapred.Task$ValuesIterator.readNextValue(Task.java:738) at org.apache.hadoop.mapred.Task$ValuesIterator.next(Task.java:678) at org.apache.hadoop.mapred.Task$CombineValuesIterator.next(Task.java:757) at WordCountReducer.reduce(WordCountReducer.java:24) at WordCountReducer.reduce(WordCountReducer.java:1) at org.apache.hadoop.mapred.MapTask$MapOutputBuffer.combineAndSpill(MapTask.java:904) at org.apache.hadoop.mapred.MapTask$MapOutputBuffer.sortAndSpill(MapTask.java:785) at org.apache.hadoop.mapred.MapTask$MapOutputBuffer.flush(MapTask.java:698) at org.apache.hadoop.mapred.MapTask.run(MapTask.java:228) at org.apache.hadoop.mapred.TaskTracker$Child.main(TaskTracker.java:2209) Caused by: java.lang.NoSuchMethodException: CompositeGroupKey.<init>() at java.lang.Class.getConstructor0(Unknown Source) at java.lang.Class.getDeclaredConstructor(Unknown Source) at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:74) 

我知道編碼不是很好，但是現在我們不知道哪里出了問題，因此歡迎任何幫助！

Answer 1

您必須在鍵類CompositeGroupKey提供一個空的默認構造函數。 它用於序列化。

只需添加：

public CompositeGroupKey() {
}

Answer 2

每當您看到一些例外情況時，如下所示

java.lang.RuntimeException: java.lang.NoSuchMethodException: CompositeGroupKey.<init>()  

然后對象實例化就會出現問題，這意味着可能沒有一個構造函數。
默認構造函數OR
參數化構造函數
除非明確聲明，否則在編寫參數化構造函數JVM的那一刻，它將抑制默認構造函數。

RusIan Ostafiichuk給出的答案足以回答您的查詢，但我添加了更多要點，以使事情更清楚。