Error - Hadoop Word Count Program in MapReduce

Question

I am new to Hadoop to so pardon me if this looks like silly question.

I am running my below MapReduce program and getting the following error:

java.lang.Exception: java.io.IOException: Type mismatch in key from map: expected org.apache.hadoop.io.Text, recieved org.apache.hadoop.io.LongWritable at org.apache.hadoop.mapred.LocalJobRunner$Job.run(LocalJobRunner.java:354) Caused by: java.io.IOException: Type mismatch in key from map: expected org.apache.hadoop.io.Text, recieved org.apache.hadoop.io.LongWritable at org.apache.hadoop.mapred.MapTask$MapOutputBuffer.collect(MapTask.java:1019)

Any help is appreciated.

public class WordCount {

// Mapper Class
public static class MapperClass extends Mapper<Object, Text, Text, IntWritable>{


    private final static IntWritable one = new IntWritable(1);
    private Text word = new Text();

    // Mapper method defined
    public void mapperMethod(Object key,Text lineContent,Context context){
        try{
        StringTokenizer strToken = new StringTokenizer(lineContent.toString());
        //Iterating through the line 
        while(strToken.hasMoreTokens()){
            word.set(strToken.nextToken());
            try{
            context.write(word, one);
            }
            catch(Exception e){
                System.err.println(new Date()+"  ---> Cannot write data to hadoop in Mapper.");
                e.printStackTrace();
            }
        }
    }
    catch(Exception ex){
        ex.printStackTrace();
    }
    }
}
// Reducer Class
public static class ReducerClass extends Reducer<Text, IntWritable, Text, IntWritable>{

    private IntWritable result = new IntWritable();

    //Reducer method
    public void reduce(Text key,Iterable<IntWritable> values,Context context){
        try{
        int sum=0;
        for(IntWritable itr : values){
            sum+=itr.get();
        }
        result.set(sum);
        try {
            context.write(key,result);
        } catch (Exception e) {
            System.err.println(new Date()+" ---> Error while sending data to Hadoop in Reducer");
            e.printStackTrace();
        }

    }
    catch (Exception err){
        err.printStackTrace();

    }
}

}


public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
    try{
    Configuration conf = new Configuration();
    String [] arguments = new GenericOptionsParser(conf, args).getRemainingArgs();
    if(arguments.length!=2){
        System.err.println("Enter both and input and output location.");
        System.exit(1);
    }
    Job job = new Job(conf,"Simple Word Count");

    job.setJarByClass(WordCount.class);
    job.setMapperClass(MapperClass.class);
    job.setReducerClass(ReducerClass.class);


    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);



    FileInputFormat.addInputPath(job, new Path(arguments[0]));
    FileOutputFormat.setOutputPath(job, new Path(arguments[1]));

    System.exit(job.waitForCompletion(true) ? 0 : 1);
}
    catch(Exception e){

    }
}

}

Answer 1

You need to override Map method in the Mapper Class instead you have a new method. Comming to your error, as you dont have map method overridden your program boils down to a reduce only job. Reducer is getting input as LongWritable,Text but you have declared Intwritable and text as input.

Hope this explains.

public static class Map extends MapReduceBase implements Mapper<LongWritable, Text, Text, IntWritable> {
         private final static IntWritable one = new IntWritable(1);
         private Text word = new Text();

         public void map(LongWritable key, Text value, OutputCollector<Text, IntWritable> output, Reporter reporter) throws IOException {
       String line = value.toString();
       StringTokenizer tokenizer = new StringTokenizer(line);
           while (tokenizer.hasMoreTokens()) {
         word.set(tokenizer.nextToken());
             output.collect(word, one);
           }
         }
       }