使用Jsoup抓取网页数据

Jsoup是一款Java的HTML解析器，可直接解析某个URL地址、HTML文本内容。它提供了一套非常省力的API，可通过DOM，CSS以及类似于jQuery的操作方法来取出和操作数据。

基本了解参考中文文档：http://www.open-open.com/jsoup/

下面介绍一个具体例子：

比如我们要抓取：http://datacenter.mep.gov.cn/report/air_daily/air_dairy.jsp上某个时段的所有城市的天气信息并且保存为csv格式的文件。

在Chrome下先执行一次访问，查看页面访问参数：

其中得到请求方式、URL、参数列表等信息。

然后根据文档开始编写代码。

项目结构如下：

CityWeather.java:

package bean;

public class CityWeather 
{
    public final static int ITEM_NUMBER = 6;
    private String id;
    private String city;
    private String date;
    private String aqi;
    private String aql;
    private String pp;
    
    public CityWeather(){}
    
    public CityWeather(String id, String city, String date, 
            String aqi, String aql, String pp)
    {
        this.id = id;
        this.city = city;
        this.date = date;
        this.aqi = aqi;
        this.aql = aql;
        this.pp = pp;
    }
    
    public String getId() 
    {
        return id;
    }
    
    public void setId(String id) 
    {
        this.id = id;
    }
    
    public String getCity() 
    {
        return city;
    }
    
    public void setCity(String city) 
    {
        this.city = city;
    }
    
    public String getDate() 
    {
        return date;
    }
    
    public void setDate(String date) 
    {
        this.date = date;
    }
    
    public String getAqi() 
    {
        return aqi;
    }
    
    public void setAqi(String aqi) 
    {
        this.aqi = aqi;
    }
    
    public String getAql() 
    {
        return aql;
    }
    
    public void setAql(String aql) 
    {
        this.aql = aql;
    }
    
    public String getPp() 
    {
        return pp;
    }
    
    public void setPp(String pp) 
    {
        this.pp = pp;
    }
    
    @Override
    public String toString()
    {
        return id + ", " + city + ", " + date + ", " + aqi +
                ", " + aql + ", " + pp + "
";
    }
}

War.java:

package bean;

public class War 
{
    //定义常量
    public final static int GET = 0;
    public final static int POST = 1;
    //待访问网址的url
    private String url;
    //参数列表
    private String[] params;
    //对应参数值
    private String[] values;
    //请求类型，默认为GET
    private int requestMethod;
    //待获取元素的选择器语法表示
    private String selector;
    
    public War(String url, String[] params, String[] values, 
            int requestMethod, String selector)
    {
        this.url = url;
        this.params = params;
        this.values = values;
        this.requestMethod = requestMethod;
        this.setSelector(selector);
    }
    
    public String getUrl()
    {
        return url;
    }
    
    public void setUrl(String url)
    {
        this.url = url;
    }
    
    public String[] getParams()
    {
        return params;
    }
    
    public void setParams(String[] params)
    {
        this.params = params;
    }
    
    public String[] getValues()
    {
        return values;
    }
    
    public void setValues(String[] values)
    {
        this.values = values;
    }
    
    public int getRequestMethod()
    {
        return requestMethod;
    }
    
    public void setRequestMethod(int requestMethod)
    {
        this.requestMethod = requestMethod;
    }

    public String getSelector() 
    {
        return selector;
    }

    public void setSelector(String selector) 
    {
        this.selector = selector;
    }
}

Op.java:

package io;
import java.io.BufferedWriter;
import java.io.FileWriter;
import java.io.IOException;
import java.io.Writer;
import java.util.ArrayList;
import bean.CityWeather;

public class Op 
{
    public static void List2File(ArrayList<CityWeather> list, String file) throws IOException
    {
          Writer w = new FileWriter(file, true);
        BufferedWriter buffWriter = new BufferedWriter(w);
        for(CityWeather cw : list)
        {
            buffWriter.write(cw.toString());
        }
        buffWriter.close();
        w.close();
    }
}

Crawler.java:

package service;
import bean.CityWeather;
import bean.War;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.jsoup.Connection;
import java.io.IOException;
import java.util.ArrayList;

public class Crawler 
{
    public static ArrayList<CityWeather> getResults(War war) throws IOException  
    {
        Connection conn = Jsoup.connect(war.getUrl());
        if(war.getParams() != null)
        {
            for(int i = 0; i < war.getParams().length; i++)
            {
                conn.data(war.getParams()[i], war.getValues()[i]);
            }
        }
        Document doc = null;
        if(war.getRequestMethod() == War.GET)
        {
            doc = conn.timeout(10000).get();
        }
        else
        {
            doc = conn.timeout(10000).post();
        }
        if(doc == null)
        {
            System.out.print("unavailable
");
            return null;
        }
        Elements results = doc.select(war.getSelector());
        ArrayList<CityWeather> list = new ArrayList<CityWeather>();
        int i = 0;
        CityWeather cw = null;
        for(Element e: results)
        {
            switch(i % CityWeather.ITEM_NUMBER)
            {
            case 0:
                cw = new CityWeather();
                cw.setId(e.text());
                break;
            case 1:
                cw.setCity(e.text());
                break;
            case 2:
                cw.setDate(e.text());
                break;
            case 3:
                cw.setAqi(e.text());
                break;
            case 4:
                cw.setAql(e.text());
                break;
            default:
                cw.setPp(e.text().replace(',', '|'));
                list.add(cw);
            }
            i++;
        }
        return list;
    }
}

Test.java:

package test;
import java.text.DateFormat;
import java.util.ArrayList;
import java.util.Date;
import service.Crawler;
import bean.CityWeather;
import bean.War;
import io.Op;

public class Test 
{
    public static void main(String[] args) throws Exception
    {
        System.out.println(DateFormat.getDateTimeInstance().format(new Date()) + " 开始运行");
        String url = "http://datacenter.mep.gov.cn/report/air_daily/air_dairy.jsp";
        String[] params = { "city", "startdate", "enddate", "page" };
        String[] values = { "", "2016-05-01", "2016-05-01", "2" };
        int requestMethod = War.GET;
        String selector = "td.report1_6";
        War war = new War(url, params, values, requestMethod, selector);
        ArrayList<CityWeather> list = Crawler.getResults(war);
        Op.List2File(list, "data/file.csv");
        System.out.println(DateFormat.getDateTimeInstance().format(new Date()) + " 运行结束");
    }
}

经过测试：抓取14年到16年的天气信息，跑的非常慢...大概跑了一天？...

而且访问有时候会失败，需要对ArrayList<CityWeather> list = Crawler.getResults(war);循环调用直到list的size不为0表示访问成功才可以访问下一页。

没有异常处理...

我在考虑是不是可以多线程同时访问一页最后再把结果merge起来，这样肯定快很多吧。

有空实现一下。

今天试了一下，开了1000个线程网站就暂时性bug了...

看来还要适当控制一下...果然多线程、分布式之类的还需要花时间仔细研究一下。

代码下载：https://github.com/SplayHuo/JavaProject