HDFS的JAVA客户端编写

            <?
            xml version="1.0" encoding="UTF-8"
            ?>
            <?
            xml-stylesheet type="text/xsl" href="configuration.xsl"
            ?>
            <!--
             Licensed under the Apache License, Version 2.0 (the "License"); you 
    may not use this file except in compliance with the License. You may obtain 
    a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless 
    required by applicable law or agreed to in writing, software distributed 
    under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES 
    OR CONDITIONS OF ANY KIND, either express or implied. See the License for 
    the specific language governing permissions and limitations under the License. 
    See accompanying LICENSE file. 
            -->
            <!--
             Put site-specific property overrides in this file. 
            -->
            <
            configuration
            >
            <!--
             指定HADOOP所使用的文件系统schema（URI），HDFS的老大（NameNode）的地址 
            -->
            <
            property
            >
            <
            name
            >fs.defaultFS</name><value>hdfs://192.168.230.134:9000</value></property><!-- 指定hadoop运行时产生文件的存储目录 --><property><name>hadoop.tmp.dir</name><value>/home/hadoop/app/hadoop-2.4.1/data</value></property></configuration>

            <?
            xml version="1.0" encoding="UTF-8"
            ?>
            <?
            xml-stylesheet type="text/xsl" href="configuration.xsl"
            ?>
            <!--
             Licensed under the Apache License, Version 2.0 (the "License"); you 
    may not use this file except in compliance with the License. You may obtain 
    a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless 
    required by applicable law or agreed to in writing, software distributed 
    under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES 
    OR CONDITIONS OF ANY KIND, either express or implied. See the License for 
    the specific language governing permissions and limitations under the License. 
    See accompanying LICENSE file. 
            -->
            <!--
             Put site-specific property overrides in this file. 
            -->
            <
            configuration
            >
            <
            property
            >
            <
            name
            >dfs.replication</name><value>1</value></property></configuration>

            package
             cn.darrenchan.hadoop.hdfs;


            import
             java.io.FileInputStream;

            import
             java.io.FileNotFoundException;

            import
             java.io.IOException;

            import
             java.net.URI;


            import
             org.apache.commons.io.IOUtils;

            import
             org.apache.hadoop.conf.Configuration;

            import
             org.apache.hadoop.fs.FSDataOutputStream;

            import
             org.apache.hadoop.fs.FileStatus;

            import
             org.apache.hadoop.fs.FileSystem;

            import
             org.apache.hadoop.fs.LocatedFileStatus;

            import
             org.apache.hadoop.fs.Path;

            import
             org.apache.hadoop.fs.RemoteIterator;

            import
             org.junit.Before;

            import
             org.junit.Test;


            /**
            
 * 如果涉及权限用户不是hadoop然后报错的话，可以在run as->run
 * configuration->arguments里面添加-DHADOOP_USER_NAME=hadoop
 * 还有另外一种终极解决方案，就是在init()方法中 FileSystem.get进行user的指定
 * 
 * 
            @author
             chenchi
 * 
 
            */
            public
            class
             HdfsUtils {

    
            private
             FileSystem fs;

    
            /**
            
     * 加上@Before就会在test方法之前执行
     * 
     * 
            @throws
             Exception
     
            */
            
    @Before
    
            public
            void init() throws Exception {
        // 读取classpath下的xxx-site.xml 配置文件，并解析其内容，封装到conf对象中
        Configuration conf = new Configuration();
        // 如果把配置文件放到src下面，这行代码可以不用写
        // 也可以在代码中对conf中的配置信息进行手动设置，会覆盖掉配置文件中的读取的值
        conf.set("fs.defaultFS", "hdfs://192.168.230.134:9000");
        // 这一行是在windows下运行才用到的
        System.setProperty("hadoop.home.dir", "E:\\大数据教程Hadoop8天\\hadoop-2.4.1");
        // 根据配置信息，去获取一个具体文件系统的客户端操作实例对象
        fs = FileSystem.get(new URI("hdfs://weekend110:9000"), conf, "hadoop");
    }

    /**
     * 上传文件（底层写法）
     * 
     * @throws IOException
     */
    @Test
    publicvoid upload() throws IOException {
        Path dst = new Path("hdfs://192.168.230.134:9000/aa/qingshu.txt");
        FSDataOutputStream out = fs.create(dst);
        FileInputStream in = new FileInputStream("E:/qingshu.txt");
        IOUtils.copy(in, out);
    }

    /**
     * 上传文件（简单版）
     * 
     * @throws IOException
     */
    @Test
    publicvoid upload2() throws IOException {
        fs.copyFromLocalFile(new Path("e:/qingshu.txt"), new Path(
                "hdfs://weekend110:9000/aaa/bbb/ccc/qingshu100.txt"));
    }

    /**
     * 下载文件
     * 
     * @throws IOException
     * @throws IllegalArgumentException
     */
    @Test
    publicvoid download() throws IllegalArgumentException, IOException {
        // 上面是这种写法会报空指针异常的错，所以采用下面这种写法
        // fs.copyToLocalFile(new Path("hdfs://weekend110:9000/aa/qingshu.txt"),
        // new Path("e:/chen.txt"));
        fs.copyToLocalFile(false, new Path(
                "hdfs://weekend110:9000/aa/qingshu.txt"), new Path(
                "e:/chen.txt"), true);
    }

    /**
     * 查看文件信息
     * 
     * @throws IOException
     * @throws IllegalArgumentException
     * @throws FileNotFoundException
     */
    @Test
    publicvoid listFiles() throws FileNotFoundException,
            IllegalArgumentException, IOException {
        // listFiles列出的是文件信息，而且提供递归遍历
        RemoteIterator<LocatedFileStatus> files = fs.listFiles(new Path("/"),
                true);

        while (files.hasNext()) {

            LocatedFileStatus file = files.next();
            Path filePath = file.getPath();
            String fileName = filePath.getName();
            System.out.println(fileName);

        }

        System.out.println("---------------------------------");

        // listStatus 可以列出文件和文件夹的信息，但是不提供自带的递归遍历
        FileStatus[] listStatus = fs.listStatus(new Path("/"));
        for (FileStatus status : listStatus) {

            String name = status.getPath().getName();
            System.out.println(name + (status.isDirectory() ? " 目录" : " 文件"));

        }
    }

    /**
     * 创建文件夹
     * 
     * @throws IOException
     * @throws IllegalArgumentException
     */
    @Test
    publicvoid mkdir() throws IllegalArgumentException, IOException {
        // 这里也可以简写成/aaa/bbb/ccc
        fs.mkdirs(new Path("hdfs://weekend110:9000/aaa/bbb/ccc"));
    }

    /**
     * 删除文件或者文件夹
     * 
     * @throws IOException
     * @throws IllegalArgumentException
     */
    @Test
    publicvoid rm() throws IllegalArgumentException, IOException {
        // 递归删除，路径名称可以简写
        fs.delete(new Path("/aa"), true);
        // fs.rename(path, path1);// 这是移动文件到不同文件夹    }
}