使用okhttp工具集来开发:(如果文件已经存在会报错)
package com.etl; import java.io.IOException; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import okhttp3.Call; import okhttp3.Callback; import okhttp3.OkHttpClient; import okhttp3.Request; import okhttp3.Response; public class LinkHttp { private static Configuration conf = null; private static String fsName = "fs.defaultFS"; private static String fsURI = null; public static void main(String[] args) throws Exception { String name = args[0]; String uri = args[1]; String url = args[2]; String targetFile = args[3]; //文件全路径 //初始化 init(name, uri); OkHttpClient client = new OkHttpClient(); final Request request = new Request.Builder().url(url).get().build(); Call call = client.newCall(request); call.enqueue(new Callback() { @Override public void onFailure(Call call, IOException e) { System.out.println("Fail"); } @Override public void onResponse(Call call, Response response) throws IOException { FileSystem fs = null; try { Path dstPath = new Path(targetFile); fs = FileSystem.get(conf); FSDataOutputStream outputStream = fs.create(dstPath); if(response.isSuccessful()) { // System.out.println(response.body().string()); outputStream.write(response.body().bytes()); outputStream.close(); System.out.println("create file " + targetFile + " success!"); //fs.close(); } }catch (Exception e){ e.printStackTrace(); }finally { fs.close(); } System.out.println("run writeHdfs end"); //关闭 if(response.body()!=null) { response.body().close(); } } }); } private static void init(String name, String uri) { if(StringUtils.isNotBlank(fsName)){ fsName = name; } fsURI = uri; conf = new Configuration(); conf.set(fsName, fsURI); } }
配置启动脚本如下:
#!/bin/sh name=fs.defaultFS #固定不变 uri=dwpro-name1:8020 #hdfs文件系统地址 url=http://www.cnblogs.com/30go/ #待保存的http地址 targetPath=/tmp/test/king.txt # 目标的文件名 java -Djava.ext.dirs=lib com.etl.LinkHttp ${name} ${uri} ${url} ${targetPath} >> test.log 2>&1 &