zoukankan      html  css  js  c++  java
  • HDFS学习笔记(2)hdfs_shell & JavaAPI

    FileSystem shell指令

    appendToFile
    cat
    checksum
    chgrp
    chmod
    chown
    copyFromLocal
    copyToLocal
    count
    cp
    createSnapshot
    deleteSnapshot
    df
    du
    dus
    expunge
    find
    get
    getfacl
    getfattr
    getmerge
    help
    ls
    lsr
    mkdir
    moveFromLocal
    moveToLocal
    mv
    put
    renameSnapshot
    rm
    rmdir
    rmr
    setfacl
    setfattr
    setrep
    stat
    tail
    test
    text
    touchz
    truncate
    usage

    FileSystem java API

    測试代码

    package hdfs;
    
    import java.io.FileInputStream;
    import java.io.FileOutputStream;
    import java.net.URI;
    import org.apache.hadoop.conf.Configuration;
    import org.apache.hadoop.fs.FSDataInputStream;
    import org.apache.hadoop.fs.FSDataOutputStream;
    import org.apache.hadoop.fs.FileStatus;
    import org.apache.hadoop.fs.FileSystem;
    import org.apache.hadoop.fs.Path;
    import org.apache.hadoop.io.IOUtils;
    
    public class HdfsOperator {
        static final String PATH = "hdfs://Master:9000/";
        static final String DIR = "/d1";
        static final String FILE = "/d1/hello";
        static final String loadFile = "F:/readme.txt";
        static final String downFile = "F:/readme_tmp.txt";
        private static void getList(FileSystem fileSystem, String DIR) throws Exception {
            final FileStatus[] listStatus = fileSystem.listStatus(new Path(DIR));
            for(FileStatus fileStatus : listStatus) {
                String isDir = fileStatus.isDirectory() ? "dir":"file";
                final String permission = fileStatus.getPermission().toString();
                final short replication = fileStatus.getReplication();
                final long len = fileStatus.getLen();
                final String path = fileStatus.getPath().toString();
                System.out.println(isDir+"	"+permission+"	"+replication+"	"+len+"	"+path);
            }
        }
        private static void getData(FileSystem fileSystem, String FILE) throws Exception {
            final FSDataInputStream in = fileSystem.open(new Path(FILE));
            final FileOutputStream out = new FileOutputStream(downFile);
            IOUtils.copyBytes(in, out, 4096, true);
            System.out.println("get hdfs: " +FILE+ " success! " + "saved as"+ downFile);
        }
        private static void putData(FileSystem fileSystem, String src, String dest ) throws Exception {
            final FSDataOutputStream out = fileSystem.create(new Path(dest));
            final FileInputStream in = new FileInputStream(src);
            IOUtils.copyBytes(in, out, 4096, true);
        }
        private static void remove(FileSystem fileSystem, String DIR) throws Exception {
            boolean flag = fileSystem.delete(new Path(DIR), true);
            System.out.println("del " + DIR + (flag ?

    " Success":" failure") ); } private static void create(FileSystem fileSystem, String DIR) throws Exception { if( fileSystem.exists(new Path(DIR)) == true) { remove(fileSystem, DIR); } fileSystem.mkdirs(new Path(DIR)); } private static FileSystem getFileSystem(String PATH) throws Exception { return FileSystem.get(new URI(PATH), new Configuration()); } public static void main(String[] args) throws Exception { FileSystem fileSystem = getFileSystem(PATH); create(fileSystem, DIR); putData(fileSystem, loadFile, FILE); getData(fileSystem, FILE); getList(fileSystem, PATH); remove(fileSystem, FILE); } }

  • 相关阅读:
    洛谷 U141580 简化罗波切问题
    洛谷 U141578 维修电路
    洛谷 U140760 狭义公因子
    CF75C Modified GCD
    算法题-求解斐波那切数列的第N个数是几?
    算法题-求N的阶乘
    JAVA8新特性
    nginx启动脚本,手动编辑
    javah生成带有包名的头文件
    Matlab图像处理(03)-基本概念
  • 原文地址:https://www.cnblogs.com/yfceshi/p/7131528.html
Copyright © 2011-2022 走看看