zoukankan html css js c++ java

hadoop 使用java操作hdfs

1、创建目录

 1 import java.io.IOException;  
 2 import org.apache.hadoop.conf.Configuration;  
 3 import org.apache.hadoop.fs.FileSystem;  
 4 import org.apache.hadoop.fs.Path;  
 5   
 6 public class MakeDir {  
 7     public static void main(String[] args) throws IOException {  
 8         FileSystem fs = FileSystem.get(new URI("hdfs://linux1:9000"),
 9                 new Configuration(),"root");  
10         Path path = new Path("/user/hadoop/data/20130709");  
11         fs.mkdirs(path);  
12         fs.close();  
13     }  
14 }

2、删除目录

 1 import java.io.IOException;  
 2 import org.apache.hadoop.conf.Configuration;  
 3 import org.apache.hadoop.fs.FileSystem;  
 4 import org.apache.hadoop.fs.Path;  
 5   
 6 public class DeleteDir {  
 7     public static void main(String[] args) throws IOException {  
 8         Configuration conf = new Configuration();  
 9         FileSystem fs = FileSystem.get(new URI("hdfs://linux1:9000"),conf,"root");  
10           
11         Path path = new Path("/user/hadoop/data/20130710");  
12         fs.deleteOnExit(path);  
13         fs.close();  
14     }  
15 }

3、写文件

 1 import java.io.IOException;  
 2 import org.apache.hadoop.conf.Configuration;  
 3 import org.apache.hadoop.fs.FSDataOutputStream;  
 4 import org.apache.hadoop.fs.FileSystem;  
 5 import org.apache.hadoop.fs.Path;  
 6   
 7 public class WriteFile {  
 8     public static void main(String[] args) throws IOException {  
 9         FileSystem fs = FileSystem.get(new URI("hdfs://linux1:9000"),
10                 new Configuration(),"root");  
11         Path path = new Path("/user/hadoop/data/write.txt");  
12         FSDataOutputStream out = fs.create(path);  
13         out.writeUTF("da jia hao,cai shi zhen de hao!");  
14         fs.close();  
15     }  
16 }

4、读文件

 1 import java.io.IOException;  
 2 import org.apache.hadoop.conf.Configuration;  
 3 import org.apache.hadoop.fs.FSDataInputStream;  
 4 import org.apache.hadoop.fs.FileStatus;  
 5 import org.apache.hadoop.fs.FileSystem;  
 6 import org.apache.hadoop.fs.Path;  
 7   
 8 public class ReadFile {  
 9     public static void main(String[] args) throws IOException {  
10         FileSystem fs = FileSystem.get(new URI("hdfs://linux1:9000"),
11                 new Configuration(),"root");  
12         Path path = new Path("/user/hadoop/data/write.txt");  
13           
14         if(fs.exists(path)){  
15             FSDataInputStream is = fs.open(path);  
16             FileStatus status = fs.getFileStatus(path);  
17             byte[] buffer = new byte[Integer.parseInt(String.valueOf(status.getLen()))];  
18             is.readFully(0, buffer);  
19             is.close();  
20             fs.close();  
21             System.out.println(buffer.toString());  
22         }  
23     }  
24 }

5、上传本地文件到HDFS

 1 import java.io.IOException;  
 2 import org.apache.hadoop.conf.Configuration;  
 3 import org.apache.hadoop.fs.FileSystem;  
 4 import org.apache.hadoop.fs.Path;  
 5   
 6 public class CopyFromLocalFile {  
 7   
 8     public static void main(String[] args) throws IOException {  
 9         FileSystem fs = FileSystem.get(new URI("hdfs://linux1:9000"),
10                 new Configuration(),"root");  
11         Path src = new Path("/home/hadoop/word.txt");  
12         Path dst = new Path("/user/hadoop/data/");  
13         fs.copyFromLocalFile(src, dst);  
14         fs.close();  
15     }  
16 }

6、删除文件

 1 import java.io.IOException;  
 2 import org.apache.hadoop.conf.Configuration;  
 3 import org.apache.hadoop.fs.FileSystem;  
 4 import org.apache.hadoop.fs.Path;  
 5   
 6 public class DeleteFile {  
 7   
 8     public static void main(String[] args) throws IOException {  
 9         Configuration conf = new Configuration();  
10         FileSystem fs = FileSystem.get("hdfs://linux1:9000"),
11                 new Configuration(),"root");  
12         Path path = new Path("/user/hadoop/data/word.txt");  
13         fs.delete(path);  
14         fs.close();  
15     }  
16 }

 1 import java.io.IOException;  
 2 import org.apache.hadoop.conf.Configuration;  
 3 import org.apache.hadoop.fs.FileStatus;  
 4 import org.apache.hadoop.fs.FileSystem;  
 5 import org.apache.hadoop.fs.Path;  
 6     
 7 public class GetAllChildFile {  
 8     static Configuration conf = new Configuration();  
 9               
10     public static void main(String[] args)throws IOException {  
11          FileSystem fs = FileSystem.get("hdfs://linux1:9000"),
12                  new Configuration(),"root");  
13          Path path = new Path("/user/hadoop");  
14          getFile(path,fs);  
15          //fs.close();  
16      }  
17            public static void getFile(Path path,FileSystem fs) throws IOException {  
18                    FileStatus[] fileStatus = fs.listStatus(path);  
19          for(int i=0;i<fileStatus.length;i++){  
20              if(fileStatus[i].isDir()){  
21                 Path p = new Path(fileStatus[i].getPath().toString());  
22                  getFile(p,fs);  
23              }else{  
24                 System.out.println(fileStatus[i].getPath().toString());  
25              }  
26          }  
27      }  
28    
29  }

8、查找某个文件在HDFS集群的位置

 1 import java.io.IOException;  
 2 import org.apache.hadoop.conf.Configuration;  
 3 import org.apache.hadoop.fs.BlockLocation;  
 4 import org.apache.hadoop.fs.FileStatus;  
 5 import org.apache.hadoop.fs.FileSystem;  
 6 import org.apache.hadoop.fs.Path;  
 7 import org.apache.hadoop.hdfs.DistributedFileSystem;  
 8 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;  
 9   
10 public class FindFile {  
11       
12     public static void main(String[] args) throws IOException {   
13         getFileLocal();  
14     }  
15       
16     /** 
17      * 查找某个文件在HDFS集群的位置 
18      * @Title:   
19      * @Description:  
20      * @param  
21      * @return  
22      * @throws 
23      */  
24     public static void getFileLocal() throws IOException{  
25         FileSystem fs = FileSystem.get("hdfs://linux1:9000"),
26                 new Configuration(),"root");  
27         Path path = new Path("/user/hadoop/data/write.txt");  
28           
29         FileStatus status = fs.getFileStatus(path);  
30         BlockLocation[] locations = fs.getFileBlockLocations(status, 0, status.getLen());  
31           
32         int length = locations.length;  
33         for(int i=0;i<length;i++){  
34             String[] hosts = locations[i].getHosts();  
35             System.out.println("block_" + i + "_location:" + hosts[i]);  
36         }  
37     }  
38       
39 }

9、HDFS集群上所有节点名称信息

 1 package com.hadoop.file;  
 2   
 3 import java.io.IOException;  
 4 import org.apache.hadoop.conf.Configuration;  
 5 import org.apache.hadoop.fs.BlockLocation;  
 6 import org.apache.hadoop.fs.FileStatus;  
 7 import org.apache.hadoop.fs.FileSystem;  
 8 import org.apache.hadoop.fs.Path;  
 9 import org.apache.hadoop.hdfs.DistributedFileSystem;  
10 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;  
11   
12 public class FindFile {  
13       
14     public static void main(String[] args) throws IOException {   
15         getHDFSNode();  
16     }  
17       
18     /** 
19      * HDFS集群上所有节点名称信息 
20      * @Title:   
21      * @Description:  
22      * @param  
23      * @return  
24      * @throws 
25      */  
26     public static void getHDFSNode() throws IOException{  
27         Configuration conf = new Configuration();  
28         FileSystem fs = FileSystem.get("hdfs://linux1:9000"),
29                 new Configuration(),"root");  
30   
31         DistributedFileSystem  dfs = (DistributedFileSystem)fs;  
32         DatanodeInfo[] dataNodeStats = dfs.getDataNodeStats();  
33           
34         for(int i=0;i<dataNodeStats.length;i++){  
35             System.out.println("DataNode_" + i + "_Node:" + dataNodeStats[i].getHostName());  
36         }  
37           
38     }  
39       
40       
41 }

查看全文

相关阅读:
String
Array常用方法
 Array类
 ruby调试/练习时的小技巧
 集合类对象的遍历处理办法
 Tech road one step 7-Noc to 13-Nov
Tech road one step 31-Oct 6-Nov
TechRoad_oneStep_17-23 10
TechRoad_oneStep_1001
TechRoad_oneStep_0919

原文地址：https://www.cnblogs.com/xiaoaofengyue/p/8182722.html