[b0014] HDFS 常用JAVA 操作实战

目的:

学习用java进行的常用hdfs操作

参考：

[b0002] Hadoop HDFS cmd常用命令练手

环境：

hadoop2.6.4

win7 下的eclipse环境调试已经配置好，参考前面的文章

代码：

1. 创建文件夹

package hdfs;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

/**
 * 
 * @author Administrator
 *  创建文件夹,如果不存在
 */
public class CreateFolder {
    
    public static void main(String[] args) throws IOException {
        Configuration conf =new Configuration();
        conf.set("fs.defaultFS", "hdfs://ssmaster:9000/");
        FileSystem fs =  FileSystem.get(conf) ;
        Path path = new Path("/output");
        
        if(! fs.exists(path)){
            fs.mkdirs(path);
        }
    }
}

以流的方式下载文件

package hdfs;

import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;

import org.apache.commons.compress.utils.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

/**
 *  功能：      将 hdfs://ssmaster:9000/data/paper.txt下载到Windows下c:paper.txt
 *  调用方式：windows下执行,eclipse中执行
 */

public class Down_Load {

    public static void main(String[] args) {
        
    Configuration conf =new Configuration();
    conf.set("fs.defaultFS", "hdfs://ssmaster:9000/");
    
    FileSystem fs = null;
    Path src = null;
    FSDataInputStream in = null;
    FileOutputStream out = null;
      
    src = new Path("hdfs://ssmaster:9000/data/paper.txt" );
    
    try {
        
      fs = FileSystem.get(conf) ;
      in = fs.open(src);

       } catch (IOException e) {
        e.printStackTrace(); 
    }
    
    try {
        out = new FileOutputStream ("c:\paper.txt"); //等效  c:/paper.txt
    } catch (FileNotFoundException e) {
        e.printStackTrace();
    }
    
    try {
        IOUtils.copy(in, out);
    } catch (IOException e) {
        e.printStackTrace();
    }

}
}

2 上传文件

package hdfs;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

/**
 * 
 * @author Administrator
 *  上传本地文件
 */
public class UploadFile {
    
    public static void main(String[] args) throws IOException {
        Configuration conf =new Configuration();
        conf.set("fs.defaultFS", "hdfs://ssmaster:9000/");
        FileSystem fs =  FileSystem.get(conf) ;
        Path path = new Path("/output");
        Path src  = new Path("c:/paper.txt");
        
        fs.copyFromLocalFile(false, true, src, path);

    }
}

3 下载文件

package hdfs;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

/**
 * 
 * @author Administrator
 *  上传本地文件
 */
public class DownFile {
    
    public static void main(String[] args) throws IOException {
        Configuration conf =new Configuration();
        conf.set("fs.defaultFS", "hdfs://ssmaster:9000/");
        FileSystem fs =  FileSystem.get(conf) ;
        Path hdfs = new Path("/output/paper.txt");
        Path win7  = new Path("c:/paper_download.txt");
        
        fs.copyToLocalFile(hdfs, win7);

    }
}

4 删除文件

package hdfs;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

/**
 * 
 * @author Administrator
 *  删除hdfs文件,如何文件不存在，也运行正常
 */
public class DeleteFile {
    
    public static void main(String[] args) throws IOException {
        Configuration conf =new Configuration();
        conf.set("fs.defaultFS", "hdfs://ssmaster:9000/");
        FileSystem fs =  FileSystem.get(conf) ;
        Path hdfs = new Path("/output/paper.txt");
        fs.delete(hdfs, true);
        
    }
}

5 显示目录信息

package hdfs;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

/**
 * 
 * @author Administrator
 *  显示某个目录下的文件
 */
public class ListFiles {
    
    public static void main(String[] args) throws IOException {
        Configuration conf =new Configuration();
        conf.set("fs.defaultFS", "hdfs://ssmaster:9000/");
        FileSystem fs =  FileSystem.get(conf) ;
        Path hdfs = new Path("/");
        
        
        FileStatus [] files = fs.listStatus(hdfs);
        for (FileStatus file:files) {
          
           System.out.print(file.getPath().getName());
           System.out.print("	"+ file.isDirectory());
           System.out.print("	"+ file.getOwner());           
           System.out.print("
");
        }

    }
}

总结：

HDFS JAVA API 调用初步学会使用。该篇章翻过去，后续用什么再学

后续：

   有空将文件系统的常用操作实现，搜索、递归显示、查看文件内容

参考:

 1 hadoop 2.x hdfs api