在 Reducer 中使用 System.setProperty()

Question

我正在嘗試在我的 Reducer java.util.Arrays.useLegacyMergeSort設置 Java 系統屬性java.util.Arrays.useLegacyMergeSort ，以強制系統使用 JDK 6 實現 Arrays.sort 方法，而不是 JDK8。

package scoring.devicestatus;

import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.log4j.Logger;

import domain.DeviceEvent;
import domain.DeviceStatus;
import domain.EndOfPeriodEvent;
import domain.PiidBoundaryEvent;
import util.DateUtils;

public class DeviceStatusReducer extends Reducer<Text, Text, Text, NullWritable> {

    public static final Logger logger = Logger.getLogger(DeviceStatusReducer.class);

    @Override
    public void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
        -->Configuration conf = context.getConfiguration();
        .       .
        .
        try{
            .
            .
            writeToContext(key, records, context);
        }catch(Throwable t){
            logger.error("Error processing VIN: " + key.toString(), t);
            throw new RuntimeException(t);
        }
    }

    List<DeviceStatus> reduce(Iterable<Text> values, String endDate) {
        List<DeviceEvent> events = createEvents(values, endDate);
        .
        .       
    }

    List<DeviceEvent> createEvents(Iterable<Text> values, String endDate) {
        List<DeviceEvent> events = new ArrayList<DeviceEvent>(10);
        for(Text text : values){
            List<DeviceEvent> instances = DeviceEvent.getInstance(text);
            for(DeviceEvent instance : instances){
                if(!Constants.ORPHAN_PIID.equals(instance.getProgramInstanceId())){
                    events.add(instance);
                }
            }
        }
    --> System.out.println("Reducer Class:"+System.getProperty("java.util.Arrays.useLegacyMergeSort")+" "+conf.get("java.util.Arrays.useLegacyMergeSort"));
        Collections.sort(events);
        return events;
    }

    public void writeToContext(Text key, List<DeviceStatus> deviceStatuses, Context context) throws IOException, InterruptedException {
        String vin = key.toString();
        for (DeviceStatus status : deviceStatuses) {
            context.write(new Text(status.toCsvString(vin)), NullWritable.get());
        }
    }
}

---

package domain;
    public abstract class DeviceEvent implements Comparable<DeviceEvent> {

        .
        .
        .   
        @Override
        public int compareTo(DeviceEvent arg0) {
            int comparison = getTimeStamp().compareTo(arg0.getTimeStamp());
            if(comparison == 0){
                comparison = isInstalled() ? 1 : -1;
            }
            return comparison;
        }
        .
        .
        .

    }

我使用 -D 選項通過命令行運行 jar 作為

hadoop jar jarname.jar classname -Djava.util.Arrays.useLegacyMergeSort=true args

減速器確實將屬性帶入了配置，但我的錯誤仍未得到修復。 當我使用System.out.println("Reducer Class:" +System.getProperty("java.util.Arrays.useLegacyMergeSort")+ " " +conf.get("java.util.Arrays.useLegacyMergeSort"));

它看起來像這樣：

stdout: Reducer Class:null true Reducer Class:null true Reducer Class:null true Reducer Class:null true Reducer Class:null true Reducer Class:null true

Below is the stack trace: SysLog: 2016-10-05 16:25:21,032 ERROR [main] DeviceStatusReducer: Error processing VIN: 19XXXXXXXXXXX java.lang.IllegalArgumentException: Comparison method violates its general contract! at java.util.ComparableTimSort.mergeLo(ComparableTimSort.java:744) at java.util.ComparableTimSort.mergeAt(ComparableTimSort.java:481) at java.util.ComparableTimSort.mergeCollapse(ComparableTimSort.java:406) at java.util.ComparableTimSort.sort(ComparableTimSort.java:213) at java.util.Arrays.sort(Arrays.java:1312) at java.util.Arrays.sort(Arrays.java:1506) at java.util.ArrayList.sort(ArrayList.java:1454) at java.util.Collections.sort(Collections.java:141) at DeviceStatusReducer.createEvents(DeviceStatusReducer.java:139) at DeviceStatusReducer.reduce(DeviceStatusReducer.java:43) at DeviceStatusReducer.reduce(DeviceStatusReducer.java:34) at DeviceStatusReducer.reduce(DeviceStatusReducer.java:24) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:627) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:389) at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:164) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1657) at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:158)

我確信java.util.Arrays.useLegacyMergeSort可以工作，因為當我在 Main 類中將其設置為System.setProperty("java.util.Arrays.useLegacyMergeSort","true")它可以在我的日食（一個 JVM）上工作但不適用於 Hadoop 集群（多個 JVM）。 我還嘗試將它直接在 Reducer 中設置為System.setProperty("java.util.Arrays.useLegacyMergeSort","true") 。

如何讓該屬性在所有 JVM 上傳播？ 或者如何將配置屬性設置為系統屬性？ 謝謝

Answer 1

它通過將 jar 運行為

hadoop jar JarName.jar MainClassName -D mapreduce.reduce.java.opts=-Djava.util.Arrays.useLegacyMergeSort=true args

請注意，-D 后面有一個空格用於 hadoop 屬性，而 -D 后面沒有空格用於設置 JVM 屬性。