Eclipse+hadoop伪态式分布+API
1、在eclipse中新建一个Maven文件，下载对应jar包
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>2.9.2</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>2.9.2</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>2.9.2</version>
</dependency>
2、编写脚本代码（浏览hdfs目录）
public class Test {
public static void main(String[] args) throws IOException {
// TODO Auto-generated method stub
String dir = "hdfs://192.168.80.128:9000/";
try {
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(URI.create(dir),conf);
FileStatus[] status = fs.listStatus(new Path(dir));
List<String> names = new ArrayList<String>();
for (int i = 0; i < status.length; ++i) {
if (status[i].isFile()) {
names.add(status[i].getPath().toString());
System.out.println(status[i].getPath().toString());
}else if (status[i].isDirectory()) {
names.add(status[i].getPath().toString());
System.out.println(status[i].getPath().toString());
}
}
fs.close();
} catch (Exception e) {
// TODO: handle exception
}
}
}
3、如果出现一下报错
在src下创建文件log4j.properties
log4j.rootLogger=WARN, stdout
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d %p [%c] - %m%n
4、即可使用java编程来对hadoop进行操作
API操作
1、浏览hdfs目录文件（文件夹和文件）
public class ListFileAndFolder {
public static void main(String[] args) throws IOException {
// TODO Auto-generated method stub
String dir = "hdfs://192.168.80.128:9000/";//地址
try {
Configuration conf = new Configuration();//加载配置文件
FileSystem fs = FileSystem.get(URI.create(dir),conf);//获取文件系统实例
FileStatus[] status = fs.listStatus(new Path("/test"));//待获取目录
List<String> names = new ArrayList<String>();//存放获取的目录链接
for (int i = 0; i < status.length; ++i) {
if (status[i].isFile()) {//文件
names.add(status[i].getPath().toString());
System.out.println(status[i].getPath().toString());
}else if (status[i].isDirectory()) {//文件夹
names.add(status[i].getPath().toString());
System.out.println(status[i].getPath().toString());
}
}
fs.close();
} catch (Exception e) {
// TODO: handle exception
System.out.println("失败");
}
}
}
2、创建文件夹
public class CreateFolder {
public static void main(String[] args) {
// TODO Auto-generated method stub
String uri = "hdfs://192.168.80.128:9000/";//端口9000
try {
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(URI.create(uri),conf,"root");//使用root用户才有权限
String pathString = "/wujf";//文件夹名
boolean exists = fs.exists(new Path(pathString));
if(!exists){//文件夹不存在则创建
boolean result = fs.mkdirs(new Path(pathString));
System.out.println(result);
}
fs.close();
} catch (Exception e) {
// TODO: handle exception
System.out.println("失败");
}
}
}
3、创建文件并且写入内容
public class WriteFile {
public static void main(String[] args) {
String uri = "hdfs://192.168.80.128:9000/";
try {
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(URI.create(uri),conf,"root");//root用户
String pathString = "/wujf/wujf"+".txt";//路径以及文件名
FSDataOutputStream fsDataOutputStream = fs.create(new Path(pathString));
String inpuString = "I am wujf";//写入的内容
IOUtils.copyBytes(new ByteArrayInputStream(inpuString.getBytes()),fsDataOutputStream,conf,true);
fs.close();
} catch (Exception e) {
// TODO: handle exception
System.out.println("失败");
}
}
}
4、复制文件
public class CopyFile {
public static void main(String[] args) {
// TODO Auto-generated method stub
String uri = "hdfs://192.168.80.128:9000/";
try {
Configuration conf = new Configuration();
//root用户才有权限创建文件
FileSystem fs = FileSystem.get(URI.create(uri),conf,"root");
String pathString = "/output/write1"+".txt";//复制后的文件路径以及文件名
FSDataOutputStream fsDataOutputStream = fs.create(new Path(pathString));
String inpuString = "/input/test1.txt";//待复制的文件
IOUtils.copyBytes(new ByteArrayInputStream(inpuString.getBytes()),fsDataOutputStream,conf,true);
//参数1：输入流，参数2：输出流；参数3：配置对象，参数4：是否关闭流
fs.close();
} catch (Exception e) {
// TODO: handle exception
System.out.println("失败");
}
}
}
5、读取文件内容
public class ReadFile {
public static void main(String[] args) {
// TODO Auto-generated method stub
String uri = "hdfs://192.168.80.128:9000/";
try {
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(URI.create(uri),conf,"root");
String pathString = "/output/write2"+".txt";//待读取的文件
FSDataInputStream fsDataInputStream = fs.open(new Path(pathString));
IOUtils.copyBytes(fsDataInputStream, System.out, conf ,true);
fs.close();
} catch (Exception e) {
// TODO: handle exception
System.out.print("失败");
}
}
}
6、删除文件
public class DeleteFolder {
public static void main(String[] args) {
// TODO Auto-generated method stub
String uri = "hdfs://192.168.80.128:9000/";
try {
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(URI.create(uri),conf,"root");
String pathString = "/output/write"+".txt";
System.out.print(fs.deleteOnExit(new Path(pathString)));
fs.close();
} catch (Exception e) {
// TODO: handle exception
System.out.print("失败");
}
}
}
7、上传本地文件
public class UploadFile {
public static void main(String[] args) {
// TODO Auto-generated method stub
String uri = "hdfs://192.168.80.128:9000/";
try {
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(URI.create(uri),conf,"root");//root用户权限高，不容易出现其他问题
Path srcPath = new Path("D:\temp\abc.txt");//本地文件路径
Path upPath=new Path("/");//上传文件的保存位置,存放于根目录
fs.copyFromLocalFile(srcPath, upPath);
} catch (Exception e) {
// TODO: handle exception
System.out.println("失败");
}
}
}
8、下载文件到本地
public class DownloadFile {
public static void main(String[] args) {
// TODO Auto-generated method stub
String uri = "hdfs://192.168.80.128:9000/";
try {
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(URI.create(uri),conf,"root");//root用户权限高，不容易出现其他问题
Path inputStream = new Path("/write.txt");//待下载文件
Path outputStream = new Path("D:\temp");//存放路径
//参数1：是否删除源文件，参数2：待下载文件路径，参数3：存放路径，参数4:是否开启文件校验
fs.copyToLocalFile(false, inputStream, outputStream, true);
} catch (Exception e) {
// TODO: handle exception
System.out.println("失败");
}
}
}