zoukankan      html  css  js  c++  java
  • HDFS学习笔记(2)hdfs_shell & JavaAPI

    FileSystem shell指令

    appendToFile
    cat
    checksum
    chgrp
    chmod
    chown
    copyFromLocal
    copyToLocal
    count
    cp
    createSnapshot
    deleteSnapshot
    df
    du
    dus
    expunge
    find
    get
    getfacl
    getfattr
    getmerge
    help
    ls
    lsr
    mkdir
    moveFromLocal
    moveToLocal
    mv
    put
    renameSnapshot
    rm
    rmdir
    rmr
    setfacl
    setfattr
    setrep
    stat
    tail
    test
    text
    touchz
    truncate
    usage

    FileSystem java API

    測试代码

    package hdfs;
    
    import java.io.FileInputStream;
    import java.io.FileOutputStream;
    import java.net.URI;
    import org.apache.hadoop.conf.Configuration;
    import org.apache.hadoop.fs.FSDataInputStream;
    import org.apache.hadoop.fs.FSDataOutputStream;
    import org.apache.hadoop.fs.FileStatus;
    import org.apache.hadoop.fs.FileSystem;
    import org.apache.hadoop.fs.Path;
    import org.apache.hadoop.io.IOUtils;
    
    public class HdfsOperator {
        static final String PATH = "hdfs://Master:9000/";
        static final String DIR = "/d1";
        static final String FILE = "/d1/hello";
        static final String loadFile = "F:/readme.txt";
        static final String downFile = "F:/readme_tmp.txt";
        private static void getList(FileSystem fileSystem, String DIR) throws Exception {
            final FileStatus[] listStatus = fileSystem.listStatus(new Path(DIR));
            for(FileStatus fileStatus : listStatus) {
                String isDir = fileStatus.isDirectory() ? "dir":"file";
                final String permission = fileStatus.getPermission().toString();
                final short replication = fileStatus.getReplication();
                final long len = fileStatus.getLen();
                final String path = fileStatus.getPath().toString();
                System.out.println(isDir+"	"+permission+"	"+replication+"	"+len+"	"+path);
            }
        }
        private static void getData(FileSystem fileSystem, String FILE) throws Exception {
            final FSDataInputStream in = fileSystem.open(new Path(FILE));
            final FileOutputStream out = new FileOutputStream(downFile);
            IOUtils.copyBytes(in, out, 4096, true);
            System.out.println("get hdfs: " +FILE+ " success! " + "saved as"+ downFile);
        }
        private static void putData(FileSystem fileSystem, String src, String dest ) throws Exception {
            final FSDataOutputStream out = fileSystem.create(new Path(dest));
            final FileInputStream in = new FileInputStream(src);
            IOUtils.copyBytes(in, out, 4096, true);
        }
        private static void remove(FileSystem fileSystem, String DIR) throws Exception {
            boolean flag = fileSystem.delete(new Path(DIR), true);
            System.out.println("del " + DIR + (flag ?

    " Success":" failure") ); } private static void create(FileSystem fileSystem, String DIR) throws Exception { if( fileSystem.exists(new Path(DIR)) == true) { remove(fileSystem, DIR); } fileSystem.mkdirs(new Path(DIR)); } private static FileSystem getFileSystem(String PATH) throws Exception { return FileSystem.get(new URI(PATH), new Configuration()); } public static void main(String[] args) throws Exception { FileSystem fileSystem = getFileSystem(PATH); create(fileSystem, DIR); putData(fileSystem, loadFile, FILE); getData(fileSystem, FILE); getList(fileSystem, PATH); remove(fileSystem, FILE); } }

  • 相关阅读:
    第三方组件之SinGooCMS.Plugins
    .net 工具集之SinGooCMS.Utility
    轻量级ORM之SinGooCMS.ADO
    Mustache 使用说明
    npm install -g 错误集锦
    获取Rtx用户状态方法
    sudo 运行 npm或node时,出现找不到命令
    install build essential
    安装hadoop 2.2.0
    异步更新界面方法
  • 原文地址:https://www.cnblogs.com/yfceshi/p/7131528.html
Copyright © 2011-2022 走看看