Hadoop MapReduce reducer无法启动

Question

The map phase runs and then just quits without bothering with the reducer. The job alternately prints "Hello from mapper." and "Writing CellWithTotalAmount" and that's it. The output directory it creates is empty.

I've checked at least a dozen of other "reducer won't start" questions and have not found an answer. I've checked that the output of map is the same as input into reduce, that reduce uses Iterable, that correct output classes have been set, etc.

Job config

public class HoursJob {
    public static void main(String[] args) throws Exception {
        if (args.length != 2) {
          System.err.println("Usage: HoursJob <input path> <output path>");
          System.exit(-1);
        }

        Job job = Job.getInstance();
        job.setJarByClass(HoursJob.class);
        job.setJobName("Hours job");

        FileInputFormat.addInputPath(job, new Path(args[0]));
        FileOutputFormat.setOutputPath(job, new Path(args[1]));

        job.setMapperClass(HoursMapper.class);
        job.setReducerClass(HoursReducer.class);

        job.setMapOutputKeyClass(IntWritable.class);
        job.setMapOutputValueClass(CellWithTotalAmount.class);

        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(NullWritable.class);

        int ret = job.waitForCompletion(true) ? 0 : 1;
        System.exit(ret);
    }
}

Mapper

public class HoursMapper 
        extends Mapper<LongWritable, Text, IntWritable, CellWithTotalAmount> {
    static double BEGIN_LONG = -74.913585;
    static double BEGIN_LAT = 41.474937;
    static double GRID_LENGTH = 0.011972;
    static double GRID_HEIGHT = 0.008983112;

    @Override
    public void map(LongWritable key, Text value, Mapper.Context context)
            throws IOException, InterruptedException {

        System.out.println("Hello from mapper.");
        String recordString = value.toString();
        try {
            DEBSFullRecord record = new DEBSFullRecord(recordString);
            Date pickupDate = record.getPickup();
            Calendar calendar = GregorianCalendar.getInstance();
            calendar.setTime(pickupDate);
            int pickupHour = calendar.get(Calendar.HOUR_OF_DAY);
            int cellX = (int)
                ((record.getPickupLongitude() - BEGIN_LONG) / GRID_LENGTH) + 1;
            int cellY = (int)
                ((BEGIN_LAT - record.getPickupLatitude()) / GRID_HEIGHT) + 1;

            CellWithTotalAmount hourInfo = 
                new CellWithTotalAmount(cellX, cellY, record.getTotal());
            context.write(new IntWritable(pickupHour), hourInfo);
        } catch (Exception ex) {
            System.out.println(
                "Cannot parse: " + recordString + "due to the " + ex);
        }
    }
}

Reducer

public class HoursReducer 
        extends Reducer<IntWritable, CellWithTotalAmount, Text, NullWritable> {
    @Override
    public void reduce(IntWritable key, Iterable<CellWithTotalAmount> values, 
            Context context) throws IOException, InterruptedException {
        System.out.println("Hello from reducer.");
        int[][] cellRideCounters = getCellRideCounters(values);
        CellWithRideCount cellWithMostRides = 
            getCellWithMostRides(cellRideCounters);

        int[][] cellTotals = getCellTotals(values);
        CellWithTotalAmount cellWithGreatestTotal = 
            getCellWithGreatestTotal(cellTotals);

        String output = key + " "
            + cellWithMostRides.toString() + " "
            + cellWithGreatestTotal.toString();

        context.write(new Text(output), NullWritable.get());
    }

    //omitted for brevity
}

Custom writable class

public class CellWithTotalAmount implements Writable {
    public int cellX;
    public int cellY;
    public double totalAmount;

    public CellWithTotalAmount(int cellX, int cellY, double totalAmount) {
        this.cellX = cellX;
        this.cellY = cellY;
        this.totalAmount = totalAmount;
    }

    @Override
    public void write(DataOutput out) throws IOException {
        System.out.println("Writing CellWithTotalAmount");
        out.writeInt(cellX);
        out.writeInt(cellY);
        out.writeDouble(totalAmount);
    }

    @Override
    public void readFields(DataInput in) throws IOException {
        System.out.println("Reading CellWithTotalAmount");
        cellX = in.readInt();
        cellY = in.readInt();
        totalAmount = in.readDouble();
    }

    @Override
    public String toString() {
        return cellX + " " + cellY + " " + totalAmount;
    }
}

Answer 1

I think there is a lot of exception in reduce function so Framework can not complete the job properly

    public class HoursReducer 
            extends Reducer<IntWritable, CellWithTotalAmount, Text, NullWritable> {
        @Override
        public void reduce(IntWritable key, Iterable<CellWithTotalAmount> values, 
                Context context) throws IOException, InterruptedException {
            System.out.println("Hello from reducer.");
    try{
            int[][] cellRideCounters = getCellRideCounters(values);
       if(cellRideCounter[0].length>0){ // control it before executing it. more explanation is above
            CellWithRideCount cellWithMostRides = 
                getCellWithMostRides(cellRideCounters);



            int[][] cellTotals = getCellTotals(values);
            CellWithTotalAmount cellWithGreatestTotal = 
                getCellWithGreatestTotal(cellTotals);

            String output = key + " "
                + cellWithMostRides.toString() + " "
                + cellWithGreatestTotal.toString();

            context.write(new Text(output), NullWritable.get());


     }
   }catch(Exception e)

    e.printstack();
     return;
   {

  }


}

• add try-catch to get exceptions in reduce function
• . Return from function in catch

. Also add an if statement before calling getCellWithMostRiders(..) I think the issue is in here. Fill the if statement as you want I made a guess and fill it according to my guess change it however you want if it is not proper for you