HDFS操作

1、创建HDFS目录

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

//创建HDFS目录
public class MakeDir {
    public static void main(String[] args) throws IOException{
        Configuration configuration=new Configuration();
        FileSystem fs=FileSystem.get(configuration);
        Path path=new Path("/user/hadoop/data/");
        fs.create(path);
        fs.close();
    }
}

2、删除HDFS目录

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

public class DeleteDir {
    public static void main(String[] args) throws IOException {
        Configuration configuration=new Configuration();
        FileSystem fs=FileSystem.get(configuration);
        Path path=new Path("/usr/hadoop/data");
        fs.delete(path);
        fs.close();
    }
}

3、HDFS写文件

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

public class WriteFile {
    public static void main(String[] args) throws IOException {
        Configuration configuration = new Configuration();
        FileSystem fs = FileSystem.get(configuration);
        Path path = new Path("/user/hadoop/data/write.txt");
        FSDataOutputStream outputStream = fs.create(path);
        outputStream.writeUTF("hello");
        fs.close();
    }
}

4、HDFS读文件

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;


public class ReadFile {
    public static void main(String[] args) throws IOException {
        Configuration configuration=new Configuration();
        FileSystem fs=FileSystem.get(configuration);
        Path path=new Path("/user/hadoop/data/write.txt");
        if (fs.exists(path)) {
            FSDataInputStream is=fs.open(path);
            FileStatus fileStatus=fs.getFileStatus(path);
            byte[] buffer=new byte[Integer.parseInt(String.valueOf(fileStatus.getLen()))];
            is.readFully(0,buffer);
            is.close();
            fs.close();
            System.out.println(buffer.toString());
        }
    }
}

5、上传本地文件到HDFS

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;


public class CopyFromLocalFile {
    public static void main(String[] args) throws IOException {
        Configuration configuration=new Configuration();
        FileSystem fs=FileSystem.get(configuration);
        Path srcPath=new Path("/home/hadoop/word.txt");
        Path dsPath=new Path("/user/hadoop/data/");
        fs.copyFromLocalFile(srcPath, dsPath);
        fs.close();
    }
}

6、删除文件

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

public class DeleteFile {
    public static void main(String[] args) throws IOException {
        Configuration configuration=new Configuration();
        FileSystem fs=FileSystem.get(configuration);
        Path path=new Path("/usr/hadoop/data/word.txt");
        fs.delete(path);
        fs.close();
    }
}

7、获取给定目录下的所有子目录以及子文件

//获取给定目录下的所有子目录以及子文件 
import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;


public class GetAllChildFile {
    static Configuration configuration=new Configuration();
    public static void main(String[] args) throws IOException {
        FileSystem fs=FileSystem.get(configuration);
        Path path=new Path("/user/hadoop");
        getFile(path,fs);
    }
    
    public static void  getFile(Path path,FileSystem fs) throws IOException {
        FileStatus[] fileStatus=fs.listStatus(path);
        for (int i = 0; i < fileStatus.length; i++) {
            if (fileStatus[i].isDir()) {
                Path path2=new Path(fileStatus[i].getPath().toString());
                getFile(path2, fs);
            }
            else {
                System.out.println(fileStatus[i].getPath().toString());
            }
        }
    }
}

8、HDFS集群上所有节点名称信息

//HDFS集群上所有节点名称信息 
import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;

public class DataNodeInfo {
    public static void main(String[] args) throws IOException {
        getHDFSNode();
    }
    public static void getHDFSNode() throws IOException {
        Configuration configuration=new Configuration();
        FileSystem fileSystem=FileSystem.get(configuration);
        
        DistributedFileSystem distributedFileSystem=(DistributedFileSystem)fileSystem;
        DatanodeInfo[] datanodeInfo=distributedFileSystem.getDataNodeStats();
        for (int i = 0; i < datanodeInfo.length; i++) {
            System.out.println("DataNode_"+i+"_Node:"+datanodeInfo[i].getHostName());
        }
    }
}