简体   繁体   中英

send multiple arguments into reducer of different types

I am trying to send 2 float arguments and one int argument to the reducer from csv file . float values results are successfully loaded into HDFS but the int result not visible in the final output. and I verified that right values are coming back to context.write method but don't understand why int values are not displayed? Can any one please help me out.

public class MaxTemperature extends Configured implements Tool {

public static class MapMapper extends Mapper<LongWritable, Text, Text, PairWritable>{

    public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException{
        String regex = ",";//''single quote not applicable for comma.
        String[] val = value.toString().split(regex);

        FloatWritable[]  vv = new FloatWritable[2];
        vv[0]= new FloatWritable(Float.parseFloat(val[3]));
        vv[1]=new FloatWritable(Float.parseFloat(val[13]));

        int k= Integer.parseInt(val[17]);
        IntWritable kk = new IntWritable(k);                
        float dd=Float.parseFloat(val[3]);
        PairWritable ddd = new PairWritable();
        context.write(new Text(val[2]), ddd.set(vv[0], vv[1],kk));

    }

} 


 public static class PairWritable extends ArrayWritable implements Writable{


    public PairWritable() {
        super(FloatWritable.class);
        // TODO Auto-generated constructor stub
    }

    private FloatWritable floatone;
     private FloatWritable floattwo;
     private IntWritable intone;


     public String toString() {

            String s = Float.toString(floatone.get());
            String  a=Float.toString(floattwo.get());
            String q = s+'\t'+a;  
            return q;
          }


    public PairWritable set(float f1, float f2, int k){
         FloatWritable ff1 = new FloatWritable(f1);
         FloatWritable ff2 = new FloatWritable(f2);
         IntWritable inq = new IntWritable(k);
         set(ff1, ff2,inq);
         return this;
     }

     public PairWritable set(FloatWritable f1, FloatWritable f2, IntWritable f3){
         this.floatone=f1;
         this.floattwo=f2;
         this.intone=f3;
         return this;
     }

     public float getone(){
         return floatone.get();
     }

     public float gettwo(){
         return floattwo.get();
     }

     public int getthreeint(){
         return intone.get();
     }

     public void write(DataOutput out) throws IOException {
            out.writeFloat(floatone.get());
            out.writeFloat(floattwo.get());
            out.writeInt(intone.get());
        }

        public void readFields(DataInput in) throws IOException {
            floatone = new FloatWritable(in.readFloat());
            floattwo = new FloatWritable(in.readFloat());
            intone= new IntWritable(in.readInt());
        }

 }



public static class Mapreducers extends Reducer<Text,PairWritable, Text,PairWritable>{

    public void reduce(Text key, Iterable<PairWritable> values,Context context) throws IOException, InterruptedException{


        float sumone =0;
        float sumtwo=0;
        int sumthree=0;

        for(PairWritable dd: values){
        sumone+=dd.getone();
        sumtwo+=dd.gettwo();
        sumthree+=dd.getthreeint();

        }

        //FloatWritable result1 = new FloatWritable(sumone);
        //FloatWritable result2 = new FloatWritable(sumtwo);
        //IntWritable result3 = new IntWritable(sumthree);
        PairWritable ddd = new PairWritable();
        context.write(key, ddd.set(sumone, sumtwo,sumthree));


}

}


public int run(String[] args) throws Exception {
    Job job = new Job();
    job.setJarByClass(MaxTemperature.class);
    job.setJobName("MaxTemperature");

    Configuration conf = new Configuration();
    FileSystem fs = FileSystem.get(URI.create(args[0]), conf);

    if(fs.exists(new Path(args[1]))){
        fs.delete(new Path(args[1]),true);
    }

    FileInputFormat.addInputPath(job, new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));

    job.setMapperClass(MapMapper.class);
    job.setCombinerClass(Mapreducers.class);
    //job.setNumReduceTasks(0);
    job.setReducerClass(Mapreducers.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(PairWritable.class);

    return job.waitForCompletion(true)?0:1;
}


public static void main(String[] args) throws Exception{
    int xx =1;
    xx = ToolRunner.run(new MaxTemperature(), args);
    System.exit(xx);    
}

}

The problem here is, I haven't converted int to string in toString method.

Used:

public String toString() {
        String s = Float.toString(floatone.get());
        String  a=Float.toString(floattwo.get());
        String q = s+'\t'+a;  
        return q;
      }

Supposed to use:

  public String toString() {

            String s = Float.toString(floatone.get());
            String  a=Float.toString(floattwo.get());
            String r=Integer.toString(intone.get());
            String q = s+'\t'+a+'\t'+r;  
            return q;
          }

The technical post webpages of this site follow the CC BY-SA 4.0 protocol. If you need to reprint, please indicate the site URL or the original address.Any question please contact:yoyou2525@163.com.

 
粤ICP备18138465号  © 2020-2024 STACKOOM.COM