hadoop 操作 hdfs

 package com.picc.test;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.URI;
import java.net.URL;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FsUrlStreamHandlerFactory;
import org.apache.hadoop.fs.Path;

public class UrlCat {
   static{
       URL.setURLStreamHandlerFactory(new FsUrlStreamHandlerFactory());
   }
   public static void main(String[] args) {
       InputStream in =null;
        try {//new FileInputStream("/home/liucheng/file/student.txt");
            in=  new URL("hdfs://localhost:9000/user/liucheng/input/student.txt").openStream();
        }  catch (IOException e) {
            e.printStackTrace();
        }
        BufferedReader read = new BufferedReader(new InputStreamReader(in));
        String line=null;
        try {
            while((line=read.readLine())!=null){
                System.out.println("result:"+line.trim());
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
        try {
            read.close();
        } catch (IOException e) {
            e.printStackTrace();
        }
    run("hdfs://localhost:9000/user/liucheng/input/student.txt");
   }
   public static void run(String hdfs){
       Configuration conf = new Configuration();
       InputStream in =null;
        try {
          FileSystem fs = FileSystem.get(URI.create(hdfs), conf);
          in = fs.open(new Path(hdfs));
          BufferedReader read = new BufferedReader(new InputStreamReader(in));
            String line=null;
            try {
                while((line=read.readLine())!=null){
                    System.out.println("result:"+line.trim());
                }
            } catch (IOException e) {
                e.printStackTrace();
            }
        } catch (IOException e) {
            e.printStackTrace();
        }finally{
            try {
                in.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
   }
}

示例是读取hdfs 的Input 文件夹下的student.txt 文件。示例。

java 操作 hdfs 

 package com.picc.test;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.URI;
import java.net.URL;
import java.util.Arrays;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.FsUrlStreamHandlerFactory;
import org.apache.hadoop.fs.Path;

public class UrlCat {
   static{
       URL.setURLStreamHandlerFactory(new FsUrlStreamHandlerFactory());
   }
   
   public static void main(String[] args) {
    
     run("hdfs://localhost:9000/user/liucheng/input/file01.txt");
    //doubleCat("hdfs://localhost:9000/user/liucheng/input/student.txt");
    //   statusTest("hdfs://localhost:9000/user/liucheng/input");
   }
   public static void run(String hdfs){
       Configuration conf = new Configuration();
       InputStream in =null;
        try {
          FileSystem fs = FileSystem.get(URI.create(hdfs), conf);
          in = fs.open(new Path(hdfs));
          BufferedReader read = new BufferedReader(new InputStreamReader(in));
            String line=null;
            try {
                while((line=read.readLine())!=null){
                    System.out.println("result:"+line.trim());
                }
            } catch (IOException e) {
                e.printStackTrace();
            }
        } catch (IOException e) {
            e.printStackTrace();
        }finally{
            try {
                in.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
   }
   //FSDataInputStream extents DataInputStream extents InputStream
   /**
    *  其实doubleCat 和 getInputStreamtoText 方法需要放到一起
    *  在有异常时，把位置保存到一个properties文件中
    * @param hdfs
    */
   public static void doubleCat(String hdfs){
       Configuration conf = new Configuration();
       FSDataInputStream fsdis =null;
       try {
        FileSystem fs = FileSystem.get(URI.create(hdfs),conf);
        fsdis= fs.open(new Path(hdfs));
        long rs = fsdis.skip(1000);//跳过1000个位置
        fsdis.seek(rs);//查找rs的位子
        String result = getInputStreamtoText(fsdis);
        System.out.println("result:"+result);
    } catch (IOException e) {
        e.printStackTrace();
    }finally{
        if(fsdis!=null){
            try {
                fsdis.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }
   }
   /**
    * inputStream to Text
    * @param in
    * @return
    */
   public static String getInputStreamtoText(InputStream in ){
       BufferedReader read = new BufferedReader(new InputStreamReader(in));
        String line=null;
        StringBuffer result=new StringBuffer();
        try {
            while((line=read.readLine())!=null){
            //    System.out.println("result:"+line.trim());
                String lineSeparator =System.getProperty("line.separator");
                result.append(line).append(lineSeparator);
                
            }
        } catch (IOException e) {
            e.printStackTrace();
        }finally{
            try {
                in.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
        return result.toString();
   }
   public static void statusTest(String url){
       Configuration conf = new Configuration();
     try {
        FileSystem fs =   FileSystem.get(URI.create(url), conf);
        Path [] paths = new Path[url.length()];
        for(int i=0;i<paths.length;i++){
//            paths[i]=new Path[String.url];
            FileStatus [] filestatus = fs.listStatus(paths[i]);
            Path [] listedPaths = FileUtil.stat2Paths(filestatus);
            for(Path p:listedPaths){
                System.out.println(p);
            }
        }
         // /tmp/hadoop-liucheng/dfs/name/current/edits
    //    bin/mysqld
    } catch (IOException e) {
        e.printStackTrace();
    }
   }
   
}