zoukankan      html  css  js  c++  java
  • 搭建hadoop java开发环境

    package hadoopDemo;
    
    import java.io.IOException;
    import java.net.URI;
    import java.net.URISyntaxException;
    
    import org.apache.hadoop.conf.Configuration;
    import org.apache.hadoop.fs.FileSystem;
    import org.apache.hadoop.fs.LocatedFileStatus;
    import org.apache.hadoop.fs.Path;
    import org.apache.hadoop.fs.RemoteIterator;
    
    public class Aa {
    
        public static void main(String[] args)  throws Exception{
            Aa a=new Aa();
            
           
        } 
        public void upload(String src,String dst) throws IOException, InterruptedException, URISyntaxException{
            //拿到一个文件系统与客户端一个实例
            FileSystem filesys=FileSystem.get(new URI("hdfs://linux1:9000"),
                    new Configuration(),"root");
            filesys.copyFromLocalFile(new Path(src), new Path(dst));
            filesys.close();
        }
        public void download(String src,String dst) throws IOException, InterruptedException, URISyntaxException{
            FileSystem fileSystem = FileSystem.get(new URI("hdfs://linux1:9000"), new Configuration(), "root");
               fileSystem.copyToLocalFile(new Path(src), new Path(dst));
               fileSystem.close();
        }
        
        public void getAllFile() throws IOException, InterruptedException, URISyntaxException{
            Configuration cfg=new Configuration();
            FileSystem filesys=FileSystem.get(new URI("hdfs://linux1:9000"),cfg,"root");
            RemoteIterator<LocatedFileStatus> listFiles=filesys.listFiles(new Path("/"), true);
            
            while(listFiles.hasNext()){
                LocatedFileStatus next2=listFiles.next();
                String name=next2.getPath().getName();
                System.out.println(next2.getPath()+"  "+name);
                
            }
        }
        
        public void mkdir(String dir) throws IOException, InterruptedException, URISyntaxException{
            Configuration cfg=new Configuration();
            FileSystem filesys=FileSystem.get(new URI("hdfs://linux1:9000"),cfg,"root");
            filesys.mkdirs(new Path(dir));
            filesys.close();
        }
        public void mv(String src,String dst) throws IOException, InterruptedException, URISyntaxException{
            Configuration cfg=new Configuration();
    //        cfg.set("fs.defaultFs", "hdfs://linux1:9000");
            FileSystem filesys=FileSystem.get(new URI("hdfs://linux1:9000"),cfg,"root");
            filesys.moveFromLocalFile(new Path(src), new Path(dst));
            filesys.close();
        }
        public void delete(String path) throws IOException, InterruptedException, URISyntaxException{
            Configuration cfg=new Configuration();
            FileSystem filesys=FileSystem.get(new URI("hdfs://linux1:9000"),cfg,"root");
            filesys.delete(new Path(path), true);
            filesys.close();
        }
    }
    View Code

    1.解压hadoop-2.6.4.tar.gz

    将此bin文件夹与hadoop-2.6.4文件夹中的bin文件夹合并

    将此bin文件夹中的hadoop.dll文件拷贝到C:WindowsSystem32目录中

    配置windows环境变量

    控制面板——>系统——>更改设置——>高级——>环境变量

    新建变量:HADOOP_HOME,路径:解压文件夹位置

    Path后添加:;%HADOOP_HOME%/bin;%HADOOP_HOME%/sbin

    测试生效:

    cmd输入hadoop

    保险起见可以重启电脑

    2.创建user library

    3.添加jar包

    添加完jar包以后进行简单测试:

    package hadoopDemo;
    
    import java.io.IOException;
    import java.net.URI;
    import java.net.URISyntaxException;
    
    import org.apache.hadoop.conf.Configuration;
    import org.apache.hadoop.fs.FileSystem;
    import org.apache.hadoop.fs.LocatedFileStatus;
    import org.apache.hadoop.fs.Path;
    import org.apache.hadoop.fs.RemoteIterator;
    
    public class Aa {
    
        public static void main(String[] args)  throws Exception{
            Aa a=new Aa();
            a.mv("e:/c.txt", "/aaa"); 

           a.getAllFile();
         
         a.delete("/aaa");
         
         a.delete("/b.txt");
         }
    public void upload(String src,String dst) throws IOException, InterruptedException, URISyntaxException{ //拿到一个文件系统与客户端一个实例 FileSystem filesys=FileSystem.get(new URI("hdfs://linux1:9000"), new Configuration(),"root"); filesys.copyFromLocalFile(new Path(src), new Path(dst)); filesys.close(); } public void download(String src,String dst) throws IOException, InterruptedException, URISyntaxException{ FileSystem fileSystem = FileSystem.get(new URI("hdfs://linux1:9000"), new Configuration(), "root"); fileSystem.copyToLocalFile(new Path(src), new Path(dst)); fileSystem.close(); } public void getAllFile() throws IOException, InterruptedException, URISyntaxException{ Configuration cfg=new Configuration(); FileSystem filesys=FileSystem.get(new URI("hdfs://linux1:9000"),cfg,"root"); RemoteIterator<LocatedFileStatus> listFiles=filesys.listFiles(new Path("/"), true); while(listFiles.hasNext()){ LocatedFileStatus next2=listFiles.next(); String name=next2.getPath().getName(); System.out.println(next2.getPath()+" "+name); } } public void mkdir(String dir) throws IOException, InterruptedException, URISyntaxException{ Configuration cfg=new Configuration(); FileSystem filesys=FileSystem.get(new URI("hdfs://linux1:9000"),cfg,"root"); filesys.mkdirs(new Path(dir)); filesys.close(); } public void mv(String src,String dst) throws IOException, InterruptedException, URISyntaxException{ Configuration cfg=new Configuration(); // cfg.set("fs.defaultFs", "hdfs://linux1:9000"); FileSystem filesys=FileSystem.get(new URI("hdfs://linux1:9000"),cfg,"root"); filesys.moveFromLocalFile(new Path(src), new Path(dst)); filesys.close(); } public void delete(String path) throws IOException, InterruptedException, URISyntaxException{ Configuration cfg=new Configuration(); FileSystem filesys=FileSystem.get(new URI("hdfs://linux1:9000"),cfg,"root"); filesys.delete(new Path(path), true); filesys.close(); } }
  • 相关阅读:
    CodeBlocks "no such file or directory" 错误解决方案(创建类找不到头文件)
    WCF配置文件与文件下载之坎坷路
    使用Visual Studio 2010打造C语言编译器
    一个小程序引发的思考
    在C#使用文件监控对象FileSystemWatcher 实现数据同步
    C 语言 static、extern与指针函数介绍
    检测端口是否被占用
    C# ini文件读写类
    C学习笔记(2)--指针
    plsql auto 常用语法
  • 原文地址:https://www.cnblogs.com/xiaoaofengyue/p/8117912.html
Copyright © 2011-2022 走看看