如何使用Java将数据从SAS服务器提取到HDFS？

Question

我想从sas服务器中提取数据并将其放入HDFS中。我想对任何Idea都使用Java代码。

Answer 1

The easiest mechanism is going to be to install the Hadoop client libraries and then upload your SAS files onto HDFS via the client commands: hadoop fs -put <filename> </hdfs/path/filename>

If you don't want to use the client commands, you can always write your own java application to perform the upload from the SAS server to HDFS. An untested piece of example code that transfers a file using the fs.FileSystem.copyFromLocalFile interface:

package org.mycompany;
import java.security.PrivilegedExceptionAction;
import org.apache.hadoop.conf.*;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileStatus;

public class HDFSTransfer {

    public static void main(String args[]) {

        try {
            UserGroupInformation ugi
                = UserGroupInformation.createRemoteUser("myuser");

            ugi.doAs(new PrivilegedExceptionAction<Void>() {

                public Void run() throws Exception {

                    Configuration conf = new Configuration();
                    conf.set("fs.defaultFS", "hdfs://10.20.30.40:8020/user/myuser");
                    conf.set("hadoop.job.ugi", "myuser");

                    FileSystem fs = FileSystem.get(conf);

                    fs.copyFromLocalFile(new Path("/local/file/test"), new Path("/user/myuser/test"));

                    FileStatus[] status = fs.listStatus(new Path("/user/myuser"));
                    for(int i=0;i<status.length;i++){
                        System.out.println(status[i].getPath());
                    }
                    return null;
                }
            });
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}