zoukankan      html  css  js  c++  java
  • HDFS文件的基本操作

    HDFS文件的基本操作:

    package wjn;
    
    import java.io.BufferedInputStream;
    import java.io.BufferedReader;
    import java.io.FileInputStream;
    import java.io.FileOutputStream;
    import java.io.IOException;
    import java.io.InputStream;
    import java.io.InputStreamReader;
    import java.io.OutputStream;
    import java.net.URI;
    import java.text.SimpleDateFormat;
    import java.util.Scanner;
    
    import org.apache.hadoop.conf.Configuration;
    import org.apache.hadoop.fs.FSDataInputStream;
    import org.apache.hadoop.fs.FSDataOutputStream;
    import org.apache.hadoop.fs.FileStatus;
    import org.apache.hadoop.fs.FileSystem;
    import org.apache.hadoop.fs.LocatedFileStatus;
    import org.apache.hadoop.fs.Path;
    import org.apache.hadoop.fs.RemoteIterator;
    import org.apache.hadoop.io.IOUtils;
    import org.apache.log4j.BasicConfigurator;
    
    public class ww {
    
        public static Scanner sc = new Scanner(System.in); 
        /**
         * @param args
         * @throws IOException 
         */
        public static void main(String[] args) throws IOException {
            // TODO Auto-generated method stub
            
            main9();
        }
    
        public static void main10() throws IOException{
            String p1 = "/user/hadoop/text/text1";
            String p2 = "/user/hadoop";
            if(mv(p1,p2)){
                System.out.println("文件移动成功");
            }else{
                System.out.println("文件移动失败");
            }
        }
        
        public static void main9() throws IOException{
            String hdfspath = "/user/hadoop/text2";
            
            if(deleteFileFromHDFS(hdfspath)){
            System.out.println("已删除文件"+hdfspath);
            }else{
                System.out.println("删除文件失败");
            }
        }
        
        public static void main7() throws IOException{
            
            String hdfspath1 = "/user/hadoop";
            boolean forcedelete = false;
            if(!ifex(hdfspath1)){
                mkdir(hdfspath1);
                System.out.println("创建目录"+hdfspath1);
            }else{
                if(isempty(hdfspath1)||forcedelete){
                    rmDir(hdfspath1);
                    System.out.println("删除目录"+hdfspath1);
                }else{
                    System.out.println("目录不为空,不删除");
                }
            }
            
        }
        
        
    public static void main6() throws IOException{
            
        String hdfspath = "/user/hadoop/text2";
        String hdfspath1 = "/user/hadoop";
        if(ifex(hdfspath)){
            deleteFileFromHDFS(hdfspath);
            System.out.println("该路径存在,删除路径"+hdfspath);
        }else{
            if(!ifex(hdfspath1)){
                mkdir(hdfspath1);
                System.out.println("创建文件夹"+hdfspath1);
            }
            touchz(hdfspath);
            System.out.println("创建路径"+hdfspath);
        }
        }
        
        
        public static void main5() throws IOException{
            
            String hdfspath = "/user/hadoop";
            System.out.println("所有文件信息如下");
            lsDir(hdfspath);
        }
        
        public static void main4() throws IOException{
            String hdfspath = "/user/hadoop/text2";
            System.out.println("文件信息如下");
            ls(hdfspath);
        }
        
        
        public static void main3() throws IOException{
            String hdfspath = "/user/hadoop/text2";
            cat(hdfspath);
            System.out.println("读取完成");
        }
        
        public static void main2() throws IOException{
            String localpath = "/home/hadoop/1234.txt";
            String hdfspath = "/user/hadoop/text2";
            download(hdfspath,localpath);
            System.out.println("文件下载成功");
        }
        
        
        public static void main1() throws IOException{
            String localpath = "/home/hadoop/123.txt";
            String hdfspath = "/user/hadoop/text2";
            if(ifex(hdfspath)){
                System.out.println("文件存在,请选择追加(1)还是覆盖(2)");
                int i = sc.nextInt();
                if(i==1){
                    appendFileToHDFS(hdfspath,localpath);
                    System.out.println("文件追加成功");
                }else if(i==2){
                    deleteFileFromHDFS(hdfspath);
                    update(localpath,hdfspath);
                    System.out.println("文件覆盖成功");
                }else{
                    System.out.println("输入有误");
                }
                    
            }else{
                update(localpath,hdfspath);
                System.out.println("文件不存在,上传成功");
            }
        }
        
        
        public static void update(String localpath , String hdfspath) throws IOException{
            
            InputStream in = new BufferedInputStream(new FileInputStream(localpath));
            FileSystem fileSystem = FileSystem.get(URI.create(hdfspath), new Configuration());
            OutputStream out = fileSystem.create(new Path(hdfspath));
    
            IOUtils.copyBytes(in, out, 4096, true);
            fileSystem.close();
            
        }
        
        //判断hdfs中文件是否存在
        public static boolean  ifex(String hdfspath) throws IOException{
             
            Configuration conf = new Configuration(); 
            conf.set("fs.defaultFS","hdfs://localhost:9000"); 
            conf.set("fs.hdfs.impl","org.apache.hadoop.hdfs.DistributedFileSystem"); 
            FileSystem fs = FileSystem.get(conf); 
            if (fs.exists(new Path(hdfspath))){ 
                fs.close();
                return true;
                }else{ 
                    fs.close();
                    return false;
                } 
        }
        
        //创建目录
        public static void mkdir(String hdfspath) throws IOException{
            
            FileSystem fileSystem = FileSystem.get(URI.create(hdfspath), new Configuration());
            Path dirPath = new Path(hdfspath);
            fileSystem.mkdirs(dirPath);
            fileSystem.close();
        }
        
        //创建文件
    public static void touchz(String hdfspath) throws IOException{
            
            FileSystem fileSystem = FileSystem.get(URI.create(hdfspath), new Configuration());
            Path dirPath = new Path(hdfspath);
            FSDataOutputStream outputStream = fileSystem.create(dirPath);
            outputStream.close();
            fileSystem.close();
        }
        
        
        
        public static void appendFileToHDFS(String hdfsPath, String localFilePath) throws IOException {
            Configuration config = new Configuration();
            config.set("dfs.client.block.write.replace-datanode-on-failure.policy", "NEVER");
            config.set("dfs.client.block.write.replace-datanode-on-failure.enable", "true");
    
            FileSystem fileSystem = FileSystem.get(URI.create(hdfsPath), config);
    
            InputStream in = new BufferedInputStream(new FileInputStream(localFilePath));
            FSDataOutputStream out = fileSystem.append(new Path(hdfsPath));
    
            IOUtils.copyBytes(in, out, 4096, true);
            fileSystem.close();
        }
        
        //删除文件
        public static boolean deleteFileFromHDFS(String hdfsPath) throws IOException {
            FileSystem fileSystem = FileSystem.get(URI.create(hdfsPath), new Configuration());
            boolean result = fileSystem.deleteOnExit(new Path(hdfsPath));
            fileSystem.close();
            return result;
        }
        
        //删除目录
        public static void rmDir(String hdfspath) throws IOException{
        
            FileSystem fileSystem = FileSystem.get(URI.create(hdfspath), new Configuration());
            fileSystem.delete(new Path(hdfspath),true);
            fileSystem.close();
        }
        
        //判断目录是否为空
        public static boolean isempty(String hdfspath) throws IOException{
            
            FileSystem fileSystem = FileSystem.get(URI.create(hdfspath), new Configuration());
            RemoteIterator<LocatedFileStatus> remoteIterator = fileSystem.listFiles(new Path(hdfspath), true);
            return !remoteIterator.hasNext();
        }
        
        
        public static void download (String hdfsPath, String localPath) throws IOException {
            FileSystem fileSystem = FileSystem.get(URI.create(hdfsPath), new Configuration());
    
            FSDataInputStream in = fileSystem.open(new Path(hdfsPath));
            OutputStream out = new FileOutputStream(localPath);
    
            IOUtils.copyBytes(in, out, 4096, true);
            fileSystem.close();
        }
        
        //根据hdfs路径输出其内容到终端
        public static void cat(String hdfspath) throws IOException{
            
               FileSystem fileSystem = FileSystem.get(URI.create(hdfspath), new Configuration());
    
                FSDataInputStream in = fileSystem.open(new Path(hdfspath));
            
                BufferedReader d = new BufferedReader(new InputStreamReader(in));
                String line = null;
                while((line = d.readLine())!=null){
                    System.out.println(line);
                }
                d.close();
                in.close();
        }
        
        //显示hdfs中指定文件的信息
        public static void ls(String hdfspath) throws IOException{
            
            FileSystem fileSystem = FileSystem.get(URI.create(hdfspath), new Configuration());
            Path path = new Path(hdfspath);
            FileStatus[] fileStatus = fileSystem.listStatus(path);
            for (FileStatus s : fileStatus){
                System.out.println("路径:"+s.getPath().toString());
                System.out.println("权限:"+s.getPermission().toString());
                System.out.println("大小:"+s.getLen());
                Long time = s.getModificationTime();
                SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
                String date = format.format(time);
                System.out.println("时间:"+date);
            }
        }
        
        
        //显示文件夹下所有文件的信息
        public static void lsDir(String hdfspath) throws IOException{
            
            FileSystem fileSystem = FileSystem.get(URI.create(hdfspath), new Configuration());
            Path dirPath = new Path(hdfspath);
            RemoteIterator<LocatedFileStatus> remoteIterator = fileSystem.listFiles(dirPath, true);
            while(remoteIterator.hasNext()){
                
                FileStatus s = remoteIterator.next();
                System.out.println("路径:"+s.getPath().toString());
                System.out.println("权限:"+s.getPermission().toString());
                System.out.println("大小:"+s.getLen());
                Long time = s.getModificationTime();
                SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
                String date = format.format(time);
                System.out.println("时间:"+date);
                System.out.println();
            }
            fileSystem.close();
        }
        
        //移动文件
        public static boolean mv(String path1,String path2) throws IOException{
            
            FileSystem fileSystem = FileSystem.get(URI.create(path1), new Configuration());
            
            Path p1 = new Path(path1);
            Path p2 = new Path(path2);
            boolean result = fileSystem.rename(p1, p2);
            fileSystem.close();
            return result;
        }
        
    }
  • 相关阅读:
    day 28 客户端服务端架构介绍,互联网协议,互联网协议补充,三次握手与四次挥手
    day25 单例模式实现方法一 单例模式实现方法二
    day24 反射/定制__str__方法控制对象的打印格式/定制__del__方法回收系统资源/元类介绍以及默认元类type创造类的过程/自定义元类控制类的创建过程/自定义元类控制类的调用过程
    day 23 封装之如何隐藏、封装之隐藏属性的底层原理、封装的真正意义、property特性、绑定方法与非绑定方法
    day22 组合、菱形继承问题、C3算法与mro介绍、在子类派生的新方法中重用父类功能的两种方式、super()严格以来继承关系、多态与鸭子类型
    day21 对象定制自己独有的属性/属性查找/绑定方法/小结/继承介绍/继承与派生
    day19 正泽表达式,sys模块,打印进度条,subprocess模块
    day18 logging模块的补充/jason序列化与反序列化/json补充/pickle序列化与反序列化/time模块/datetime模块/random模块/os模块
    day17 包的使用 logging模块的基本用法
    day16 面向过程编程/模块介绍/import导入模块/ from...import导入模块/循环导入问题/模块的搜索路径/区分python文件的两种用途/软件开发的目录规范
  • 原文地址:https://www.cnblogs.com/123456www/p/11525298.html
Copyright © 2011-2022 走看看