Hadoop1.2.0开发笔记（七）

上文中本人是通过Hadoop的eclipse插件来管理HDFS文件系统的，在实际生产环境通常是用编程的方式来管理hadoop的文件系统

下面我们编程方式开发一个管理hadoop文件系统的应用

首先打开eclipse开发工具，新建Ｍap/Reduce Project项目

点击Next，填写项目名称，并且配置hadoop安装目录

新建立的项目会自动导入hadoop的相关依赖jar文件，不必我们手动添加了

接下来我们调用hadoop文件系统的API来执行文件操作，我们先建立一个文件管理工具类

public class HDFSUtil {
    
    /**
     * 从本地复制文件到HDFS
     * @param srcFilePath
     * @param dstFilePath
     * @throws IOException
     */
    public static void copyFile2HDFS(String srcFilePath,String dstFilePath){
        Configuration conf = new Configuration();

        Path src = new Path(srcFilePath);
        Path dst = new Path(dstFilePath);        
        
        FileSystem hdfs=null;
        try {
            hdfs = FileSystem.get(conf);            
            hdfs.copyFromLocalFile(src, dst);            
            FileStatus[] files = hdfs.listStatus(dst);
            if(files!=null) {
                for(int i=0;i<files.length;i++) {
                    System.out.println("the file is:"+files[i].getPath().getName());
                }
            }else {
                System.out.println("no files");
            }
        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
        finally
        {
            try {
                hdfs.close();
            } catch (IOException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            }
        }        
    }
    
    /**
     * 在HDFS上创建文件
     * @param content
     * @param dstFile
     * @throws IOException 
     */
    public static void createFileInHDFS(String content,String dstFile) {
        Configuration conf = new Configuration();
        //conf.addResource(new Path("/home/chenying/hadoop-1.2.0/conf/core-site.xml"));
        Path dst = new Path(dstFile);
        FileSystem hdfs = null;
        FSDataOutputStream out = null;
        try {            
            hdfs=FileSystem.get(conf);
            out = hdfs.create(dst);
           // out.writeBytes(content);
            out.write(content.getBytes("UTF-8"));
            out.flush();
        } catch (IOException e) {
            e.printStackTrace();            
        }finally {
            if(hdfs != null) {
                try {
                    hdfs.close();
                } catch (IOException e) {
                    // TODO Auto-generated catch block
                    e.printStackTrace();
                }
            }
            if(out != null) {
                try {
                    out.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }            
        }        
    }
    /**
     * 在HDFS上创建目录
     * @param content
     * @param dstFile
     * @throws IOException 
     */
    public static void createDirectoryInHDFS(String dstDir) {
        Configuration conf = new Configuration();
        Path dst = new Path(dstDir);
        FileSystem hdfs = null;        
        try {            
            hdfs=FileSystem.get(conf);
            hdfs.mkdirs(dst);            
        } catch (IOException e) {
            e.printStackTrace();
            //throw new IOException(e);
        }finally {
            if(hdfs != null) {
                try {
                    hdfs.close();
                } catch (IOException e) {
                    // TODO Auto-generated catch block
                    e.printStackTrace();
                }
            }
        }        
    }
    
    /**
     * 重命名文件
     * @param originalFile
     * @param newFile
     * @throws IOException
     */
    public static void renameFileInHDFS(String originalFile,String newFile)  {
        Configuration conf = new Configuration();
        Path originalPath = new Path(originalFile);
        Path newPath = new Path(newFile);        
        FileSystem hdfs=null;
        try {
            //hdfs = newPath.getFileSystem(conf);
            hdfs =FileSystem.get(conf);
            boolean isRename=hdfs.rename(originalPath, newPath);
            String result=isRename?"成功":"失败";
            System.out.println("文件重命名结果为："+result);
        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
        finally
        {
            try {
                hdfs.close();
            } catch (IOException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            }
        }        
    }
    
    /**
     * 获得文件的最后修改时间
     * @param dstFile
     * @throws IOException
     */
    public static void getFileLastModifyTime(String dstFile) {
        Configuration conf = new Configuration();
        Path dstPath = new Path(dstFile);
        FileSystem hdfs=null;
        try {
            //hdfs = dstPath.getFileSystem(conf);
            hdfs =FileSystem.get(conf);
            FileStatus file = hdfs.getFileStatus(dstPath);
            long time = file.getModificationTime();            
            System.out.println("the last modify time is : "+new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date(time)));
        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
        finally
        {
            try {
                hdfs.close();
            } catch (IOException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            }
        }        
    }
    
    /**
     * 检查文件是否存在
     * @param dstFile
     * @throws IOException
     */
    public static void checkFileIsExists(String dstFile) {
        Configuration conf = new Configuration();
        Path dstPath = new Path(dstFile);
        FileSystem hdfs=null;
        try {
            //hdfs = dstPath.getFileSystem(conf);
            hdfs =FileSystem.get(conf);
            boolean flag = hdfs.exists(dstPath);            
            System.out.println("is the file exists:"+flag);
        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
        finally
        {
            try {
                hdfs.close();
            } catch (IOException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            }
        }        
    }
    
    /**
     * 获得文件的存放位置
     * @param dstFile
     * @throws IOException
     */
    public static void getFileLocations(String dstFile){
        Configuration conf = new Configuration();
        Path dstPath = new Path(dstFile);
        FileSystem hdfs=null;
        try {
            //hdfs = dstPath.getFileSystem(conf);
            hdfs =FileSystem.get(conf);
            FileStatus file = hdfs.getFileStatus(dstPath);
            BlockLocation[] blkLocations = hdfs.getFileBlockLocations(file, 0, file.getLen());
            if(blkLocations != null) {
                int len = blkLocations.length;
                for(int i=0;i<len;i++) {
                    String[] hosts = blkLocations[i].getHosts();
                    for(String host:hosts) {
                        System.out.println("the location'host is : "+host);
                    }
                }
            }
        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
        finally
        {
            try {
                hdfs.close();
            } catch (IOException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            }
        }
    }
    
    /**
     * 删除文件
     * @throws IOException
     */
    public static void deleteFile(String dstFile){
        Configuration conf = new Configuration();
        Path dstPath = new Path(dstFile);
        FileSystem hdfs=null;
        try {
            //hdfs = dstPath.getFileSystem(conf);
            hdfs =FileSystem.get(conf);
            boolean flag = hdfs.delete(dstPath, true);////如果是目录则递归删除
            System.out.println("is deleted : "+flag);
        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
        finally
        {
            try {
                hdfs.close();
            } catch (IOException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            }
        }       
    }
}

（如果你愿意折腾，可以尝试将上面的方法改造成接口回调，本人就不做这个无谓的折腾了）

如果遇到Exception in thread "main" java.lang.IllegalArgumentException: Wrong FS: hdfs expected: file:///之类的异常，需要将${hadoop.root}/conf/目录的hadoop配置文件core-site.xml、hdfs-site.xml、mapred-site.xml拷贝到eclipse项目的bin目录里面

下面是测试类

public class HdfsClient {
    
    /**
     * 上传文件
     * @throws IOException
     */
    @Test
    public void testCopyFile2HDFS() throws IOException {
        String srcFilePath = "/home/chenying/FetchTask.java";
        String dstFilePath = "hdfs://localhost:9000/user/coder/in";
        HDFSUtil.copyFile2HDFS(srcFilePath, dstFilePath);
    }
    /**
     * 创建文件
     */
    @Test
    public void testCreateFileInHDFS()  {
        String content = "hey,Hadoop. 测试中文";
        String dstFile = "hdfs://localhost:9000/user/coder/input/hey";
        HDFSUtil.createFileInHDFS(content, dstFile);
    }
    /**
     * 重命名文件
     */
    @Test
    public void testRenameFileInHDFS() {
        String originalFile = "hdfs://localhost:9000/user/coder/input/hey";
        String newFile = "hdfs://localhost:9000/user/coder/input/hey_hadoop";
        HDFSUtil.renameFileInHDFS(originalFile, newFile);
    }
    /**
     * 获取文件的最后修改时间
     */
    @Test
    public void testGetFileLastModifyTimme() {
        String dstFile = "hdfs://localhost:9000/user/coder/input/hey_hadoop";
        HDFSUtil.getFileLastModifyTime(dstFile);
    }
    /**
     * 检查文件是否存在
     */
    @Test
    public void testCheckFileIsExists() {
        String dstFile = "hdfs://localhost:9000/user/coder/input/hey_hadoop";
        HDFSUtil.checkFileIsExists(dstFile);
    }
    /**
     * 获取文件的存放位置
     */
    @Test
    public void testGetFileLocations() {
        String dstFile = "hdfs://localhost:9000/user/coder/input/hey_hadoop";
        HDFSUtil.getFileLocations(dstFile);
    }
    /**
     * 删除文件
     */
    @Test
    public void testDeleteFile(){
        String dstFile = "hdfs://localhost:9000/user/coder/output";
        HDFSUtil.deleteFile(dstFile);
    }

}

运行该测试类，正常情况下，上面的测试用例会显示绿色

后面部分，本人打算采用HDFS开发一个图片服务器 

--------------------------------------------------------------------------- 

本系列Hadoop1.2.0开发笔记系本人原创 

转载请注明出处 博客园 刺猬的温驯  

本文链接 http://www.cnblogs.com/chenying99/archive/2013/06/02/3113449.html