zoukankan      html  css  js  c++  java
  • HDFS API基本操作

    对HDFS API基本操作都是通过 org.apache.hadoop.fs.FileSystem类进行的,以下是一些常见的操作:

    package HdfsAPI;
    
    import java.io.BufferedInputStream;
    import java.io.File;
    import java.io.FileInputStream;
    import java.io.IOException;
    import java.io.InputStream;
    import java.net.URI;
    
    import org.apache.hadoop.conf.Configuration;
    import org.apache.hadoop.fs.BlockLocation;
    import org.apache.hadoop.fs.FSDataOutputStream;
    import org.apache.hadoop.fs.FileStatus;
    import org.apache.hadoop.fs.FileSystem;
    import org.apache.hadoop.fs.Path;
    import org.apache.hadoop.io.IOUtils;
    import org.junit.Test;
    
    import junit.framework.TestCase;
    
    
    public class HDFSAPI extends TestCase{  
        public static String hdfsUrl = "hdfs://master:8020";
        //创建文件夹
        @Test
        public void testHDFSmkdir() throws IOException{
            Configuration conf = new Configuration();
            FileSystem fs = FileSystem.get(URI.create(hdfsUrl),conf);
            Path path = new Path("/liguodong/hdfs");
            fs.mkdirs(path);
        }
    
        //创建文件
        @Test
        public void testHDFSmkFile() throws IOException{
            Configuration conf = new Configuration();
            FileSystem fs = FileSystem.get(URI.create(hdfsUrl),conf);
            Path path = new Path("/liguodong/hdfs/liguodong.txt");
            FSDataOutputStream fdos = fs.create(path); 
            fdos.write("hello hadoop".getBytes());
        }
        //重命名
        @Test
        public void testHDFSRenameFile() throws IOException{
            Configuration conf = new Configuration();
            FileSystem fs = FileSystem.get(URI.create(hdfsUrl),conf);
            Path path = new Path("/liguodong/hdfs/liguodong.txt");
            Path Renamepath = new Path("/liguodong/hdfs/love.txt");
            System.out.println(fs.rename(path, Renamepath));
        }
        //上传一个本地文件
        @Test
        public void testHDFSPutLocalFile1() throws IOException{
            Configuration conf = new Configuration();
            FileSystem fs = FileSystem.get(URI.create(hdfsUrl),conf);
            //因为在windows上面调试,所以选择的是windows的文件夹,
            //假设在Linxu,须要Linux文件夹。
            Path srcpath = new Path("g:/liguodong.txt");
            Path destpath = new Path("/liguodong/hdfs");
            fs.copyFromLocalFile(srcpath, destpath);
        }
    
        @Test
        public void testHDFSPutLocalFile2() throws IOException{
            Configuration conf = new Configuration();
            FileSystem fs = FileSystem.get(URI.create(hdfsUrl),conf);
            //因为在windows上面调试,所以选择的是windows的文件夹,
            //假设在Linxu,须要Linux文件夹。
            String srcpath ="g:/oncemore.txt";
            Path destpath = new Path("/liguodong/hdfs/kkk.txt");
            InputStream is = new BufferedInputStream(
                    new FileInputStream(new File(srcpath)));
            FSDataOutputStream fdos = fs.create(destpath);
            IOUtils.copyBytes(is, fdos, 4094);
        }
    
        //查看某个文件夹以下的全部文件
        @Test
        public void testHDFSListFiles() throws IOException{
            Configuration conf = new Configuration();
            FileSystem fs = FileSystem.get(URI.create(hdfsUrl),conf);
            Path path = new Path("/liguodong/hdfs");
            FileStatus[] files = fs.listStatus(path);
    
            for (FileStatus file : files) {
                System.out.println(file.getPath().toString());
            }
        }
    
        //查看某个文件的数据块信息
        @Test
        public void testHDFSGetBlockInfo() throws IOException{
            Configuration conf = new Configuration();
            FileSystem fs = FileSystem.get(URI.create(hdfsUrl),conf);
            Path path = new Path("/liguodong/hdfs/kkk.txt");
            FileStatus filestatus = fs.getFileStatus(path);
            BlockLocation[] blkLoc = fs.getFileBlockLocations
                    (filestatus, 0, filestatus.getLen());
            for (BlockLocation loc : blkLoc) {
                for (int i = 0; i < loc.getHosts().length; i++) {
                    //获取数据块在哪些主机上
                    System.out.println(loc.getHosts()[i]);//获取文件块的主机名
                    //因为这个文件仅仅有一个块。所以输出结果为:slave2、slave1、slave5
                }
            }
        }   
    }
  • 相关阅读:
    opencv —— copyTo 设置与操作感兴趣区域(ROI)
    opencv —— src.at<Vec3b>(i, j)[0]、src.at<uchar>(i, j)、src.ptr<uchar>(i) 访问图像的单个像素
    opencv —— getTickCount、getTickFrequency 计时函数
    opencv —— setMouseCallback 响应鼠标操作事件
    opencv —— createTrackbar、getTrackbarPos 滑动条的创建和使用
    opencv —— imread、namedWindow & imshow、cvtColor、imwrite 加载、显示、修改、保存图像
    redis缓存穿透,缓存击穿,缓存雪崩问题
    NoSQL技术
    日期处理
    WIN2008中部署网站后样式及JS加载不了(转载)
  • 原文地址:https://www.cnblogs.com/mfrbuaa/p/5382414.html
Copyright © 2011-2022 走看看