zoukankan      html  css  js  c++  java
  • HDFS API 简单示例

    package com.rabbit.hadoop.hdfs;

    import org.apache.hadoop.conf.Configuration;
    import org.apache.hadoop.fs.*;
    import org.apache.hadoop.io.IOUtils;

    import java.io.*;
    import java.net.URI;

    public class TestHDFS {

    public Configuration config = null;
    public FileSystem fs = null;

    public void conn() throws IOException, InterruptedException {
    config = new Configuration(true);
    // FileSystem fs = FileSystem.get(config);
    fs = FileSystem.get(URI.create("hdfs://mycluster"),config,"bigdata");
    }

    public void mkdir() throws IOException, InterruptedException {
    this.conn();
    Path path = new Path("/user/bigdata/idea");
    fs.mkdirs(path);
    this.close();
    }

    public void upload() throws IOException, InterruptedException {
    this.conn();
    BufferedInputStream bis = new BufferedInputStream(new FileInputStream(new File("E:/a.txt")));
    Path path = new Path("/user/bigdata/idea/a.txt");
    FSDataOutputStream fsos = fs.create(path);
    IOUtils.copyBytes(bis,fsos,config,true);
    this.close();
    }

    public void download() throws IOException, InterruptedException {
    this.conn();
    Path path = new Path("/user/bigdata/idea/a.txt");
    FSDataInputStream fsis = fs.open(path);
    BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(new File("E:/a_hdfs.txt")));
    IOUtils.copyBytes(fsis, bos,config,true);
    this.close();
    }

    public void getBlocks() throws IOException, InterruptedException {
    this.conn();
    Path path = new Path("/user/bigdata/seq.txt");
    FileStatus fileStatus = fs.getFileStatus(path);
    BlockLocation[] blockLocations = fs.getFileBlockLocations(fileStatus, 0, fileStatus.getLen());

    for (BlockLocation blk : blockLocations) {
    System.out.println(blk);
    }

    FSDataInputStream fsis = fs.open(path);
    fsis.seek(1048576);
    System.out.println((char) fsis.readByte());
    System.out.println((char) fsis.readByte());
    System.out.println((char) fsis.readByte());
    System.out.println((char) fsis.readByte());
    System.out.println((char) fsis.readByte());
    System.out.println((char) fsis.readByte());
    System.out.println((char) fsis.readByte());
    System.out.println((char) fsis.readByte());
    System.out.println((char) fsis.readByte());
    System.out.println((char) fsis.readByte());
    System.out.println((char) fsis.readByte());
    System.out.println((char) fsis.readByte());
    System.out.println((char) fsis.readByte());
    System.out.println((char) fsis.readByte());

    this.close();
    }

    public void close() throws IOException {
    fs.close();
    }

    public static void main(String[] args) throws IOException, InterruptedException {
    TestHDFS test = new TestHDFS();
    // test.mkdir();
    // test.upload();
    // test.download();
    test.getBlocks();

    }

    }
  • 相关阅读:
    暑假学习笔记(一)——初识Neo4j和APICloud入门
    置信规则库学习记录——1
    博客整理——宣传文案
    博客整理——软件工程实践总结
    博客整理——Alpha版冲刺
    博客整理——事后诸葛亮
    面试被企业拒绝后还有必要再申请吗?
    MapReduce分布式算法
    水题系列二:PhoneNumbers
    水题系列一:Circle
  • 原文地址:https://www.cnblogs.com/rabbit624/p/14458662.html
Copyright © 2011-2022 走看看