API操作HDFS
1.上传文件
/*上传文件*/
@Test
public void put() throws IOException, URISyntaxException, InterruptedException {
//用于配置HDFS相关的参数
Configuration configuration=new Configuration();
//创建一个用于操作HDFS集群的FileSystem对象
FileSystem fileSystem = FileSystem.get(URI.create("hdfs://192.168.138.102:9000"), configuration, "root");
//上传文件
fileSystem.copyFromLocalFile(new Path("E:\Hello.txt"),new Path("/test"));
fileSystem.close();
}
2.下载文件
/*下载*/
@Test
public void get() throws IOException, InterruptedException {
//获取配置文件系统
Configuration configuration=new Configuration();
//创建一个用于操作HDFS集群的FileSystem对象
FileSystem fileSystem = FileSystem.get(URI.create("hdfs://192.168.138.102:9000"), configuration, "root");
//文件下载
fileSystem. copyToLocalFile(false,new Path("/test/2.txt"),new Path("E:\"),true);
//释放资源
fileSystem.close();
}
3.删除文件
/*文件删除*/
@Test
public void del() throws IOException, InterruptedException {
//获取文件系统
Configuration configuration=new Configuration();
//配置在集群上运行
FileSystem fileSystem = FileSystem.get(URI.create("hdfs://192.168.138.102:9000"), configuration, "root");
//执行删除
fileSystem.delete(new Path("/test/wang"),true);
//释放资源
fileSystem.close();
}
4.重命名文件
/*重命名文件*/
@Test
public void rname() throws IOException, InterruptedException {
//配置参数
Configuration configuration=new Configuration();
//创建操作HDFS的文件系统对象
FileSystem fileSystem = FileSystem.get(URI.create("hdfs://192.168.138.102:9000"), configuration, "root");
//修改文件名称
fileSystem.rename(new Path("/nnn"),new Path("/input"));
//关闭资源
fileSystem.close();
}
5.追加文件
/*追加文件:将一个本地文件的内容追加到HDFS文件的末尾*/
@Test
public void du() throws IOException, InterruptedException {
//配置参数
Configuration configuration=new Configuration();
//创建操作HDFS的文件系统对象
FileSystem fileSystem = FileSystem.get(URI.create("hdfs://192.168.138.102:9000"), configuration, "root");
FSDataOutputStream append = fileSystem.append(new Path("/test/2.txt"), 1024);
FileInputStream open = new FileInputStream("E:\Hello.txt");
IOUtils.copyBytes(open,append,1024,true);
fileSystem.close();
}
6.详情查看文件
/*查看文件详情*/
@Test
public void file() throws IOException, InterruptedException {
//配置参数
Configuration configuration=new Configuration();
//创建操作HDFS的文件系统对象
FileSystem fileSystem = FileSystem.get(URI.create("hdfs://192.168.138.102:9000"), configuration, "root");
//执行文件详情操作
RemoteIterator<LocatedFileStatus> listFiles = fileSystem.listFiles(new Path("/test"), true);
while (listFiles.hasNext()){
LocatedFileStatus next=listFiles.next();
//文件名称
System.out.println(next.getPath().getName());
//文件大小
System.out.println(next.getLen());
//文件权限
System.out.println(next.getPermission());
//具体块的具体信息
BlockLocation[] blockLocations=next.getBlockLocations();
for (BlockLocation bl:blockLocations){
System.out.println(bl.getOffset());
String[] hosts = bl.getHosts();
for (String host:hosts){
System.out.println(host);
}
}
System.out.println("-----------------------------");
}
fileSystem.close();
}
7.文件和文件夹判断
/*判断是文件还是文件夹*/
@Test
public void ls() throws IOException, InterruptedException {
//配置参数
Configuration configuration=new Configuration();
//创建操作HDFS的文件系统对象
FileSystem fileSystem = FileSystem.get(URI.create("hdfs://192.168.138.102:9000"), configuration, "root");
//执行查看文件详情操作
FileStatus[] fileStatuses = fileSystem.listStatus(new Path("/test"));
for (FileStatus fileStatus:fileStatuses){
if (fileStatus.isFile()){
System.out.println("以下信息是一个文件信息");
//文件名称
System.out.println(fileStatus.getPath().getName());
//文件路径
System.out.println(fileStatus.getPath());
//文件大小
System.out.println(fileStatus.getLen());
//文件权限
System.out.println(fileStatus.getPermission());
System.out.println("----------------------------------------");
}else{
System.out.println("这是一个文件夹");
System.out.println(fileStatus.getPath());
}
}
fileSystem.close();
}
IO流操作HDFS
1.IO文件上传
/*IO操作文件上传*/
@Test
public void putIO() throws IOException, InterruptedException {
//获取文件系统
Configuration configuration=new Configuration();
FileSystem fileSystem = FileSystem.get(URI.create("hdfs://192.168.138.102:9000"), configuration, "root");
//获取输出流
FSDataOutputStream str = fileSystem.create(new Path("/test/One.txt"));
//获取输入流
FileInputStream fileInputStream = new FileInputStream(new File("E:\One.txt"));
try {
//流对接
IOUtils.copyBytes(fileInputStream,str,configuration);
}finally {
//关闭资源
IOUtils.closeStream(str);
IOUtils.closeStream(fileInputStream);
}
}
2.IO文件下载
/*IO操作文件下载*/
@Test
public void getIO() throws IOException, InterruptedException {
//获取文件系统
Configuration configuration=new Configuration();
FileSystem fileSystem = FileSystem.get(URI.create("hdfs://192.168.138.102:9000"), configuration, "root");
//获取输出流
FSDataInputStream open = fileSystem.open(new Path("/test/baidu.txt"));
//创建输出流
FileOutputStream fileOutputStream = new FileOutputStream(new File("E://baidu.txt"));
try {
//流对接
IOUtils.copyBytes(open,fileOutputStream,configuration);
}finally {
IOUtils.closeStream(fileOutputStream);
IOUtils.closeStream(open);
}
}