Java程序操作HDFS

1.新建项目
2.导包

解压hadoop-2.7.3.tar.gz
E:工具大数据大数据提升资料1-软件资料6-Hadoop安装包Java1.8环境
下编译hadoop-2.7.3hadoop-2.7.3sharehadoopcommon
E:工具大数据大数据提升资料1-软件资料6-Hadoop安装包Java1.8环境
下编译hadoop-2.7.3hadoop-2.7.3sharehadoopcommonlib
E:工具大数据大数据提升资料1-软件资料6-Hadoop安装包Java1.8环境
下编译hadoop-2.7.3hadoop-2.7.3sharehadoophdfs

JUNIT  单元测试
  /**
   * 单元测试
   * 1.修饰符必须是public
   * 2.不能有参数
   * 3.不能有返回值
   *
   * 运行：选中方法名来运行
   *       run as  --junit test
   */
  @Test
  public  void m1(){
    System.out.println("hello");
  }

package com.zy.hdfs;

import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.Iterator;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.junit.Before;
import org.junit.Test;

public class test {
    public static void main(String[] args) throws Exception {
        //上传
        
        //1实例化configuration
        Configuration configuration = new Configuration();
        //-------------------
        //设置一些属性
        configuration.set("dfs.replication", "2");//2个副本
        configuration.set("dfs.blocksize", "80m");//按照80m切分
        //----------------------
        //2.获取操作文件系统的客户端实例
        FileSystem fs = FileSystem.get(new URI("hdfs://192.168.64.111:9000/"), configuration, "root");
        //3.执行上传操作  (src 本地资源 ,dst 上传目的地)
        fs.copyFromLocalFile(new Path("C:\Users\Administrator\Desktop\wc.txt"), new Path("/"));//两次上传一样的会被覆盖
        //4.关闭链接
        fs.close();
    }
    
    

    @Test
    public void download() throws Exception{
        //下载
        
        Configuration configuration = new Configuration();
        
        FileSystem fs = FileSystem.get(new URI("hdfs://192.168.64.111:9000"), configuration, "root");
        
        fs.copyToLocalFile(false, new Path("/jdk-8u141-linux-x64.tar.gz"), new Path("E://jdk-8u141-linux-x64.tar.gz"), true);
        
        fs.close();    
        
    }
    //----------------
    FileSystem fs=null;
    @Before//在每次运行junit时先执行
    public void before() throws Exception{
        System.out.println("11111");
        Configuration configuration = new Configuration();    
        fs = FileSystem.get(new URI("hdfs://192.168.64.111:9000/"), configuration, "root");        
    }
    //---------------
    @Test
    public void delete() throws Exception{//删除
        fs.delete(new Path("/弹性表达式.doc"), true);//为true删除文件夹及里面的内容，false只能删除空的文件夹
        fs.close();
        
    }
    @Test
    public void mkdir() throws Exception{//创建目录
        fs.mkdirs(new Path("/a/b/c"));
        fs.close();
        
    }
    
    @Test
    public void list() throws Exception{
        FileStatus[] listStatus = fs.listStatus(new Path("/"));//不会递归进去
        for (FileStatus fileStatus : listStatus) {
            if(fileStatus.isFile()){
                System.out.println("文件名:"+fileStatus.getPath().toString());
                
            }else{
                System.out.println("目录:"+fileStatus.getPath().toString());
                
            }
            
        }
        fs.close();
        
    }
    @Test
    public void list2() throws Exception{//递归遍历出所有文件
        RemoteIterator<LocatedFileStatus> listFiles = fs.listFiles(new Path("/"), true);//为true，递归遍历
        while(listFiles.hasNext()){
            LocatedFileStatus next = listFiles.next();
            System.out.println(next.toString());
            System.out.println("文件块尺寸"+next.getBlockSize());
         
        }

    }
 
}