毕业设计进度4

实现爬取数据并存入数据库

项目结构：

　

核心代码位于test包下的BlogPageProcessor

package com.zyk.test;

import com.zyk.dao.CnblogsDao;
import com.zyk.dao.CnblogsDaoImpl;
import com.zyk.entity.Cnblogs;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;

import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class BlogPageProcessor implements PageProcessor {

    private Site site = Site.me().setRetryTimes(10).setSleepTime(1000);

    private static int num = 0;
    CnblogsDao cnblogsDao=new CnblogsDaoImpl();

    public static void main(String[] args) throws Exception {
        long startTime ,endTime;
        System.out.println("========zyk的爬虫【启动】！=========");
        startTime = new Date().getTime();
        Spider.create(new BlogPageProcessor()).addUrl("https://www.cnblogs.com/cate/108698/").thread(200).run();
        endTime = new Date().getTime();
        System.out.println("========zyk小爬虫【结束】=========");
        System.out.println("一共爬到"+num+"篇博客！用时为："+(endTime-startTime)/1000+"s");
    }
    @Override
    public void process(Page page) {
            if(page.getUrl().regex("https://www.cnblogs.com/cate/108698/.*").match()){
                page.addTargetRequests(page.getHtml().xpath("//div[@id='post_list']").links().regex("^(.*\.html)$").all());
                page.addTargetRequests(page.getHtml().xpath("////div[@class='pager']").links().all());
            }
            else{
                try {

                    Cnblogs cnblogs=new Cnblogs();
                        //获取url
                    String url=page.getHtml().xpath("//a[@id='cb_post_title_url']/@href").get();
                    //获取标题
                    String title = page.getHtml().xpath("//a[@id='cb_post_title_url']/text()").toString();
                    //获取作者
                    String author = page.getHtml().xpath("//a[@id='Header1_HeaderTitle']/text()").toString();
                    //获取时间
                    String time=page.getHtml().xpath("//span[@id='post-date']/text()").toString();
                    //获取评论量
                    String comment = page.getHtml().xpath("////span[@id='stats_comment_count']/text()").get();

                    //获取阅读量
                    String view = page.getHtml().xpath("//div[@class='postDesc']/span[@id='post_view_count']/text()").toString();




                    cnblogs.setUrl(url);
                    cnblogs.setTitle(title);
                    cnblogs.setAuthor(author);
                    cnblogs.setTime(time);
                    if(comment==null){
                        cnblogs.setComment("评论 - 0");
                    }else{
                        cnblogs.setComment(comment);
                    }
                    cnblogs.setComment(comment);
                    cnblogs.setView(view);

                    num++;

                    cnblogsDao.saveBlog(cnblogs);

                } catch (Exception e) {
                    e.printStackTrace();
                }
            }
    }

    @Override
    public Site getSite() {
        return this.site;
    }
}

通过调用webmagic的接口就可以存入数据库中