crawler4j源码学习(1)：搜狐新闻网新闻标题采集爬虫

crawler4j是用Java实现的开源网络爬虫。提供了简单易用的接口，可以在几分钟内创建一个多线程网络爬虫。下面实例结合jsoup，采集搜狐新闻网（http://news.sohu.com/）新闻标题信息。

所有的过程仅需两步完成：

第一步：建立采集程序核心部分

/**
 * @date 2016年8月20日 上午11:52:13
 * @version
 * @since JDK 1.8
 */
public class MyCrawler extends WebCrawler {

    //链接地址过滤//
    private final static Pattern FILTERS = Pattern.compile(".*(\.(css|js|gif|jpg" + "|png|mp3|mp3|zip|gz))$");

    @Override
    public boolean shouldVisit(Page referringPage, WebURL url) {
        String href = url.getURL().toLowerCase();
        return !FILTERS.matcher(href).matches() && href.startsWith("http://news.sohu.com/");
    }

    /**
     * This function is called when a page is fetched and ready to be processed
     * by your program.
     */
    @Override
    public void visit(Page page) {
        String url = page.getWebURL().getURL();
        logger.info("URL: " + url);

        if (page.getParseData() instanceof HtmlParseData) {
            HtmlParseData htmlParseData = (HtmlParseData) page.getParseData();
            String text = htmlParseData.getText();
            String html = htmlParseData.getHtml();
            Set<WebURL> links = htmlParseData.getOutgoingUrls();

            logger.debug("Text length: " + text.length());
            logger.debug("Html length: " + html.length());
            logger.debug("Number of outgoing links: " + links.size());
            logger.info("Title: " + htmlParseData.getTitle());
            
        }
    }

}

第二步：建立采集程序控制部分

/**
 * @date 2016年8月20日 上午11:55:56
 * @version
 * @since JDK 1.8
 */
public class MyController {

    /**
     * @param args
     * @since JDK 1.8
     */
    public static void main(String[] args) {
        // TODO Auto-generated method stub

        //本地嵌入式数据库，采用berkeley DB
        String crawlStorageFolder = "data/crawl/root";
        int numberOfCrawlers = 3;

        CrawlConfig config = new CrawlConfig();
        config.setCrawlStorageFolder(crawlStorageFolder);

        /*
         * Instantiate the controller for this crawl.
         */
        PageFetcher pageFetcher = new PageFetcher(config);
        RobotstxtConfig robotstxtConfig = new RobotstxtConfig();
        RobotstxtServer robotstxtServer = new RobotstxtServer(robotstxtConfig, pageFetcher);
        CrawlController controller;
        try {
            controller = new CrawlController(config, pageFetcher, robotstxtServer);
            controller.addSeed("http://news.sohu.com/");
            controller.start(MyCrawler.class, numberOfCrawlers);
        } catch (Exception e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }

    }

}

采集结果展示：