jsoup httpclient 爬取网页并下载google图标

jsoup下载地址 http://www.jsoup.org

httpclient下载地址 http://hc.apache.org/downloads.cgi

其他jar包见附件

Crawler

package jsoup;  
  
import java.io.File;  
import java.io.FileOutputStream;  
import java.io.IOException;  
import java.io.InputStream;  
import java.util.HashMap;  
import java.util.Map;  
  
import org.apache.commons.io.FileUtils;  
import org.apache.commons.io.IOUtils;  
import org.apache.http.HttpEntity;  
import org.apache.http.HttpResponse;  
import org.apache.http.HttpStatus;  
import org.apache.http.client.methods.HttpGet;  
import org.apache.http.impl.client.DefaultHttpClient;  
import org.apache.http.params.HttpProtocolParams;  
import org.apache.http.util.EntityUtils;  
  
import com.google.api.translate.Language;  
import com.google.api.translate.Translate;  
  
/** 
 * google logo 下载程序 
 */  
public abstract class Crawler {  
  
    /** 
     * 使用google 翻译api 
     *  
     * @param en 
     * @return 
     */  
    public String translateEnToCinese(String en) {  
        Translate.setHttpReferrer("http://www.xxx.com");  
        try {  
            return Translate.execute(en, Language.ENGLISH, Language.CHINESE);  
        } catch (Exception e) {  
            e.printStackTrace();  
        }  
        return "";  
    }  
  
    /** 
     * 获取一个Map 
     *  
     * @return 
     */  
    public Map<String, Object> getMap() {  
        return new HashMap<String, Object>(0);  
    }  
  
    /** 
     * 下载文件 
     *  
     * @param url 
     *            文件http地址 
     * @param dir 
     *            目标文件 
     * @throws IOException 
     */  
    public void downloadFile(String url, String dir) throws Exception {  
        DefaultHttpClient httpClient = new DefaultHttpClient();  
        HttpProtocolParams.setUserAgent(httpClient.getParams(),  
                        "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9.1.9) Gecko/20100315 Firefox/3.5.9");  
        HttpGet httpGet = new HttpGet();  
        httpGet.setURI(new java.net.URI(url));  
          
        InputStream input = null;  
        FileOutputStream output = null;  
        try {  
            HttpResponse response = httpClient.execute(httpGet);  
            HttpEntity entity = response.getEntity();  
            input = entity.getContent();  
            File file = new File(dir);  
            output = FileUtils.openOutputStream(file);  
            IOUtils.copy(input, output);  
        } catch (Exception e){  
            e.printStackTrace();  
        } finally {  
            IOUtils.closeQuietly(output);  
            IOUtils.closeQuietly(input);  
        }  
    }  
  
    /** 
     * 处理GET请求，返回整个页面 
     *  
     * @param url 
     *            访问地址 
     * @param params 
     *            编码参数 
     * @return 
     * @throws Exception 
     */  
    public synchronized String doGet(String url, String... params)  
            throws Exception {  
        DefaultHttpClient httpClient = new DefaultHttpClient(); // 创建httpClient实例  
        HttpProtocolParams.setUserAgent(httpClient.getParams(),  
                        "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9.1.9) Gecko/20100315 Firefox/3.5.9");  
        String charset = "UTF-8";  
        if (null != params && params.length >= 1) {  
            charset = params[0];  
        }  
        HttpGet httpGet = new HttpGet(); // 创建get方法实例  
        String content = "";  
        httpGet.setURI(new java.net.URI(url));  
        try {  
            HttpResponse response = httpClient.execute(httpGet); // 执行请求，得到response对象  
            int resStatu = response.getStatusLine().getStatusCode(); // 得到返回的状态码  
            if (resStatu == HttpStatus.SC_OK) { // 200正常  
                HttpEntity entity = response.getEntity(); // 获得相应的实体  
                if (entity != null) {  
                    // 使用EntityUtils的toString方法，传递默认编码，在EntityUtils中的默认编码是ISO-8859-1  
                    content = EntityUtils.toString(entity, charset);  
                }  
            }  
        } catch (Exception e) {  
            System.out.println("访问【" + url + "】出现异常!");  
            e.printStackTrace();  
        } finally {  
            // 关闭资源  
            httpGet.abort();  
            httpClient.getConnectionManager().shutdown();  
        }  
        return content;  
    }  
}  

GoogleLogoCrawler

package jsoup;  
  
import java.io.File;  
import java.io.IOException;  
import java.util.ArrayList;  
import java.util.Date;  
import java.util.List;  
import java.util.Map;  
  
import org.apache.commons.io.FileUtils;  
import org.apache.commons.lang.StringUtils;  
import org.json.JSONArray;  
import org.json.JSONObject;  
import org.jsoup.Jsoup;  
import org.jsoup.nodes.Document;  
import org.jsoup.nodes.Element;  
import org.jsoup.select.Elements;  
  
/** 
 * google logo 下载程序 
 */  
public class GoogleLogoCrawler extends Crawler {  
      
    private static final String URL = "http://www.logocollect.com/google/year.php?key=%y&page=%p";   
  
    private static final String LOGO_URL = "http://www.logocollect.com/google/";  
  
    private static final String[] YEARS = new String[] {   
            //"1998", "1999", "2000",  
            //"2001", "2002", "2003", "2004", "2005", "2006", "2007", "2008",   
            "2009", "2010", "2011", "2012" };  
  
    private static final String INDEX = "http://www.logocollect.com/google/year.php?key=%y";   
  
    private static final String DIR_PATH = "D:\googlelogos\";  
  
    public void doStart() {  
        JSONArray array = new JSONArray();  
        for (String year : YEARS) {  
            String ind = INDEX.replaceAll("%y", year);  
            int pageCount = getPageCount(ind);  
            for (int i = 1; i < pageCount+1; i++) {  
                String url = URL.replaceAll("%y", year).replaceAll("%p", i + "");  
                String path = year + "_" + i;  
                start(url, array, DIR_PATH + path + "\", path);  
            }  
        }  
        try {  
            FileUtils.writeStringToFile(new File(DIR_PATH + "json"), array.toString(), "UTF-8");  
        } catch (IOException e) {  
            e.printStackTrace();  
        }  
        System.out.println(array);  
    }  
      
    public int getPageCount(String url) {  
        int pageCount = 1;  
        try {  
            org.jsoup.nodes.Document doc = Jsoup.connect(url).get();  
              
            String els = doc.html().toString();  
            int start = els.indexOf("总页数") + 4;  
            String temp = els.substring(start);  
            int end = temp.indexOf("，");  
            pageCount = Integer.parseInt(els.substring(start,start+end));  
            System.out.println(pageCount);  
        } catch (IOException e) {  
            e.printStackTrace();  
        }  
        return pageCount;  
    }  
  
    public void start(String url, JSONArray array, String dir, String path) {  
        try {  
            String content = super.doGet(url);  
            Document doc = Jsoup.parse(content);  
            Elements dds = doc.select(".img img");  
            List<Map<String, Object>> list = new ArrayList<Map<String, Object>>(0);  
            for (int i = 0; i < dds.size(); i++) {  
                Element img = dds.get(i);  
                String src = img.select("img").first().attr("src");  
                String title = img.select("img").first().attr("title");  
                Map<String, Object> map = super.getMap();  
                  
                map.put("url", LOGO_URL + src);  
                map.put("title", title);  
                  
                list.add(map);  
            }  
            JSONArray tempJsonArray = new JSONArray();  
            for (Map<String, Object> map : list) {  
                JSONObject jsonObject = new JSONObject();  
                String proxy = StringUtils.substringAfterLast(map.get("url")  
                        .toString(), ".");  
                long date = new Date().getTime();  
                String name = date + "." + proxy;  
                jsonObject.put("url", map.get("url").toString());  
                jsonObject.put("dir", name);  
                jsonObject.put("title", map.get("title").toString());  
                  
                // 翻译  
//              String dateZh = super.translateEnToCinese(map.get("date")  
//                      .toString());  
//              String titleZh = super.translateEnToCinese(map.get("title")  
//                      .toString());  
//              json.put("title_zh_cn", dateZh + " - " + titleZh);  
                  
                // 下载图片  
                super.downloadFile(map.get("url").toString(), dir + name);  
                tempJsonArray.put(jsonObject);  
            }  
            array.put(new JSONObject().put(path, tempJsonArray));  
        } catch (Exception e) {  
            e.printStackTrace();  
        }  
    }  
  
    public static void main(String[] args) throws Exception {  
        new GoogleLogoCrawler().doStart();  
    }  
  
}  

• other.rar 

本文转自：http://you-java.iteye.com/blog/1460271