zoukankan      html  css  js  c++  java
  • HDFS API 简单示例

    package com.rabbit.hadoop.hdfs;

    import org.apache.hadoop.conf.Configuration;
    import org.apache.hadoop.fs.*;
    import org.apache.hadoop.io.IOUtils;

    import java.io.*;
    import java.net.URI;

    public class TestHDFS {

    public Configuration config = null;
    public FileSystem fs = null;

    public void conn() throws IOException, InterruptedException {
    config = new Configuration(true);
    // FileSystem fs = FileSystem.get(config);
    fs = FileSystem.get(URI.create("hdfs://mycluster"),config,"bigdata");
    }

    public void mkdir() throws IOException, InterruptedException {
    this.conn();
    Path path = new Path("/user/bigdata/idea");
    fs.mkdirs(path);
    this.close();
    }

    public void upload() throws IOException, InterruptedException {
    this.conn();
    BufferedInputStream bis = new BufferedInputStream(new FileInputStream(new File("E:/a.txt")));
    Path path = new Path("/user/bigdata/idea/a.txt");
    FSDataOutputStream fsos = fs.create(path);
    IOUtils.copyBytes(bis,fsos,config,true);
    this.close();
    }

    public void download() throws IOException, InterruptedException {
    this.conn();
    Path path = new Path("/user/bigdata/idea/a.txt");
    FSDataInputStream fsis = fs.open(path);
    BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(new File("E:/a_hdfs.txt")));
    IOUtils.copyBytes(fsis, bos,config,true);
    this.close();
    }

    public void getBlocks() throws IOException, InterruptedException {
    this.conn();
    Path path = new Path("/user/bigdata/seq.txt");
    FileStatus fileStatus = fs.getFileStatus(path);
    BlockLocation[] blockLocations = fs.getFileBlockLocations(fileStatus, 0, fileStatus.getLen());

    for (BlockLocation blk : blockLocations) {
    System.out.println(blk);
    }

    FSDataInputStream fsis = fs.open(path);
    fsis.seek(1048576);
    System.out.println((char) fsis.readByte());
    System.out.println((char) fsis.readByte());
    System.out.println((char) fsis.readByte());
    System.out.println((char) fsis.readByte());
    System.out.println((char) fsis.readByte());
    System.out.println((char) fsis.readByte());
    System.out.println((char) fsis.readByte());
    System.out.println((char) fsis.readByte());
    System.out.println((char) fsis.readByte());
    System.out.println((char) fsis.readByte());
    System.out.println((char) fsis.readByte());
    System.out.println((char) fsis.readByte());
    System.out.println((char) fsis.readByte());
    System.out.println((char) fsis.readByte());

    this.close();
    }

    public void close() throws IOException {
    fs.close();
    }

    public static void main(String[] args) throws IOException, InterruptedException {
    TestHDFS test = new TestHDFS();
    // test.mkdir();
    // test.upload();
    // test.download();
    test.getBlocks();

    }

    }
  • 相关阅读:
    Web框架下安全漏洞的测试反思
    如何能低成本地快速获取大量目标用户,而不是与竞争对手持久战?
    Spring-Boot自定义Starter实践
    WM_QUERYENDSESSION与WM_ENDSESSION
    AutoMapper用法
    使用AutoMapper实现Dto和Model的自由转换(下)
    使用AutoMapper实现Dto和Model的自由转换(中)
    使用AutoMapper实现Dto和Model的自由转换(上)
    JSON中JObject和JArray的修改
    通信对象 System.ServiceModel.Channels.ServiceChannel 无法用于通信,因为其处于“出错”状态。
  • 原文地址:https://www.cnblogs.com/rabbit624/p/14458662.html
Copyright © 2011-2022 走看看