zoukankan      html  css  js  c++  java
  • HDFS API 简单示例

    package com.rabbit.hadoop.hdfs;

    import org.apache.hadoop.conf.Configuration;
    import org.apache.hadoop.fs.*;
    import org.apache.hadoop.io.IOUtils;

    import java.io.*;
    import java.net.URI;

    public class TestHDFS {

    public Configuration config = null;
    public FileSystem fs = null;

    public void conn() throws IOException, InterruptedException {
    config = new Configuration(true);
    // FileSystem fs = FileSystem.get(config);
    fs = FileSystem.get(URI.create("hdfs://mycluster"),config,"bigdata");
    }

    public void mkdir() throws IOException, InterruptedException {
    this.conn();
    Path path = new Path("/user/bigdata/idea");
    fs.mkdirs(path);
    this.close();
    }

    public void upload() throws IOException, InterruptedException {
    this.conn();
    BufferedInputStream bis = new BufferedInputStream(new FileInputStream(new File("E:/a.txt")));
    Path path = new Path("/user/bigdata/idea/a.txt");
    FSDataOutputStream fsos = fs.create(path);
    IOUtils.copyBytes(bis,fsos,config,true);
    this.close();
    }

    public void download() throws IOException, InterruptedException {
    this.conn();
    Path path = new Path("/user/bigdata/idea/a.txt");
    FSDataInputStream fsis = fs.open(path);
    BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(new File("E:/a_hdfs.txt")));
    IOUtils.copyBytes(fsis, bos,config,true);
    this.close();
    }

    public void getBlocks() throws IOException, InterruptedException {
    this.conn();
    Path path = new Path("/user/bigdata/seq.txt");
    FileStatus fileStatus = fs.getFileStatus(path);
    BlockLocation[] blockLocations = fs.getFileBlockLocations(fileStatus, 0, fileStatus.getLen());

    for (BlockLocation blk : blockLocations) {
    System.out.println(blk);
    }

    FSDataInputStream fsis = fs.open(path);
    fsis.seek(1048576);
    System.out.println((char) fsis.readByte());
    System.out.println((char) fsis.readByte());
    System.out.println((char) fsis.readByte());
    System.out.println((char) fsis.readByte());
    System.out.println((char) fsis.readByte());
    System.out.println((char) fsis.readByte());
    System.out.println((char) fsis.readByte());
    System.out.println((char) fsis.readByte());
    System.out.println((char) fsis.readByte());
    System.out.println((char) fsis.readByte());
    System.out.println((char) fsis.readByte());
    System.out.println((char) fsis.readByte());
    System.out.println((char) fsis.readByte());
    System.out.println((char) fsis.readByte());

    this.close();
    }

    public void close() throws IOException {
    fs.close();
    }

    public static void main(String[] args) throws IOException, InterruptedException {
    TestHDFS test = new TestHDFS();
    // test.mkdir();
    // test.upload();
    // test.download();
    test.getBlocks();

    }

    }
  • 相关阅读:
    ActiveReports中如何在后台导出运行时绑定数据源报表
    ActiveReports中如何控制页面的记录数
    WordPress建站固定链接问题
    Linux环境下使用g++编译C++
    Git diff结果显示分析
    VTK使用过程中遇到的问题
    右值引用、移动语义和完美转发(下)
    右值引用、移动语义和完美转发(中)
    右值引用、移动语义和完美转发(上)
    new和malloc的联系与区别(下)
  • 原文地址:https://www.cnblogs.com/rabbit624/p/14458662.html
Copyright © 2011-2022 走看看