用Java測試Hadoop HDFS

Question

我是Hadoop新手。

我的hadoop版本是2.8.1，我使用的是Oxygen 3a。 我正在嘗試用Java測試Hadoop HDFS庫。 我可以使用Java Eclipse上的FileSystem庫直接連接到HDFS嗎？ 即我想在Java Eclipse中而不是在cmd上測試hdfs庫 。

這是我的源代碼。 此源正在Hadoop分布式環境上工作。 但是，當我在單模式下測試時，它不起作用。

package us.qi.hdfs;

import java.net.URI;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;

public class HDFS_Test{

    public static void main(String args[]) throws Exception {
        String uri = "hdfs://mycluster:9000";
        Configuration conf = new Configuration();
        conf.set("fs.defaultFS", uri);
        conf.addResource(new Path("/home/hadoop/hadoop-2.8.1/etc/hadoop/core-site.xml"));
        conf.addResource(new Path("/home/hadoop/hadoop-2.8.1/etc/hadoop/hdfs-site.xml"));

        FileSystem fs = FileSystem.get(URI.create(uri), conf);

        Path[] paths = new Path[args.length];
        for (int i = 0;i < paths.length; i++) {
            paths[i] = new Path(args[i]);
        }

        FileStatus[] status = fs.listStatus(paths);
        Path[] listPaths = FileUtil.stat2Paths(status);
        for (Path p : listPaths) {
            System.out.println(p);
        }
    }
}

另外，在Java中調用配置對象會導致以下錯誤：

log4j:ERROR setFile(null,true) call failed.
java.io.FileNotFoundException: /datanode-metrics.log (Permission denied)
    at java.io.FileOutputStream.open0(Native Method)
    at java.io.FileOutputStream.open(FileOutputStream.java:270)
    at java.io.FileOutputStream.<init>(FileOutputStream.java:213)
    at java.io.FileOutputStream.<init>(FileOutputStream.java:133)
    at org.apache.log4j.FileAppender.setFile(FileAppender.java:294)
    at org.apache.log4j.RollingFileAppender.setFile(RollingFileAppender.java:207)
    at org.apache.log4j.FileAppender.activateOptions(FileAppender.java:165)
    at org.apache.log4j.config.PropertySetter.activate(PropertySetter.java:307)
    at org.apache.log4j.config.PropertySetter.setProperties(PropertySetter.java:172)
    at org.apache.log4j.config.PropertySetter.setProperties(PropertySetter.java:104)
    at org.apache.log4j.PropertyConfigurator.parseAppender(PropertyConfigurator.java:842)
    at org.apache.log4j.PropertyConfigurator.parseCategory(PropertyConfigurator.java:768)
    at org.apache.log4j.PropertyConfigurator.parseCatsAndRenderers(PropertyConfigurator.java:672)
    at org.apache.log4j.PropertyConfigurator.doConfigure(PropertyConfigurator.java:516)
    at org.apache.log4j.PropertyConfigurator.doConfigure(PropertyConfigurator.java:580)
    at org.apache.log4j.helpers.OptionConverter.selectAndConfigure(OptionConverter.java:526)
    at org.apache.log4j.LogManager.<clinit>(LogManager.java:127)
    at org.apache.log4j.Logger.getLogger(Logger.java:104)
    at org.apache.commons.logging.impl.Log4JLogger.getLogger(Log4JLogger.java:262)
    at org.apache.commons.logging.impl.Log4JLogger.<init>(Log4JLogger.java:108)
    at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
    at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
    at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
    at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
    at org.apache.commons.logging.impl.LogFactoryImpl.createLogFromClass(LogFactoryImpl.java:1025)
    at org.apache.commons.logging.impl.LogFactoryImpl.discoverLogImplementation(LogFactoryImpl.java:844)
    at org.apache.commons.logging.impl.LogFactoryImpl.newInstance(LogFactoryImpl.java:541)
    at org.apache.commons.logging.impl.LogFactoryImpl.getInstance(LogFactoryImpl.java:292)
    at org.apache.commons.logging.impl.LogFactoryImpl.getInstance(LogFactoryImpl.java:269)
    at org.apache.commons.logging.LogFactory.getLog(LogFactory.java:657)
    at org.apache.hadoop.conf.Configuration.<clinit>(Configuration.java:185)
    at us.qi.hdfs.HDFS_Test.main(HDFS_Test.java:16)
log4j:ERROR setFile(null,true) call failed.
java.io.FileNotFoundException: /namenode-metrics.log (Permission denied)
    at java.io.FileOutputStream.open0(Native Method)
    at java.io.FileOutputStream.open(FileOutputStream.java:270)
    at java.io.FileOutputStream.<init>(FileOutputStream.java:213)
    at java.io.FileOutputStream.<init>(FileOutputStream.java:133)
    at org.apache.log4j.FileAppender.setFile(FileAppender.java:294)
    at org.apache.log4j.RollingFileAppender.setFile(RollingFileAppender.java:207)
    at org.apache.log4j.FileAppender.activateOptions(FileAppender.java:165)
    at org.apache.log4j.config.PropertySetter.activate(PropertySetter.java:307)
    at org.apache.log4j.config.PropertySetter.setProperties(PropertySetter.java:172)
    at org.apache.log4j.config.PropertySetter.setProperties(PropertySetter.java:104)
    at org.apache.log4j.PropertyConfigurator.parseAppender(PropertyConfigurator.java:842)
    at org.apache.log4j.PropertyConfigurator.parseCategory(PropertyConfigurator.java:768)
    at org.apache.log4j.PropertyConfigurator.parseCatsAndRenderers(PropertyConfigurator.java:672)
    at org.apache.log4j.PropertyConfigurator.doConfigure(PropertyConfigurator.java:516)
    at org.apache.log4j.PropertyConfigurator.doConfigure(PropertyConfigurator.java:580)
    at org.apache.log4j.helpers.OptionConverter.selectAndConfigure(OptionConverter.java:526)
    at org.apache.log4j.LogManager.<clinit>(LogManager.java:127)
    at org.apache.log4j.Logger.getLogger(Logger.java:104)
    at org.apache.commons.logging.impl.Log4JLogger.getLogger(Log4JLogger.java:262)
    at org.apache.commons.logging.impl.Log4JLogger.<init>(Log4JLogger.java:108)
    at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
    at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
    at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
    at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
    at org.apache.commons.logging.impl.LogFactoryImpl.createLogFromClass(LogFactoryImpl.java:1025)
    at org.apache.commons.logging.impl.LogFactoryImpl.discoverLogImplementation(LogFactoryImpl.java:844)
    at org.apache.commons.logging.impl.LogFactoryImpl.newInstance(LogFactoryImpl.java:541)
    at org.apache.commons.logging.impl.LogFactoryImpl.getInstance(LogFactoryImpl.java:292)
    at org.apache.commons.logging.impl.LogFactoryImpl.getInstance(LogFactoryImpl.java:269)
    at org.apache.commons.logging.LogFactory.getLog(LogFactory.java:657)
    at org.apache.hadoop.conf.Configuration.<clinit>(Configuration.java:185)
    at us.qi.hdfs.HDFS_Test.main(HDFS_Test.java:16)
Exception in thread "main" java.lang.NoClassDefFoundError: org/apache/commons/collections/map/UnmodifiableMap
    at org.apache.hadoop.conf.Configuration$DeprecationContext.<init>(Configuration.java:416)
    at org.apache.hadoop.conf.Configuration.<clinit>(Configuration.java:456)
    at us.qi.hdfs.HDFS_Test.main(HDFS_Test.java:16)
Caused by: java.lang.ClassNotFoundException: org.apache.commons.collections.map.UnmodifiableMap
    at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
    at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:338)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
    ... 3 more

如果有人知道解決方案，請幫助我。 謝謝。

Answer 1

我通過重新布置hadoop庫來解決它。

我錯過了一些日食圖書館。 因此，我對Hadoop通用，Hadoop-HDFS使用了Maven，並且它可以正常工作。