Java爬虫学习（2）之用对象保存文件demo（1）

package com.mieba.spider;

import java.util.ArrayList;
import java.util.List;
import java.util.Vector;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Html;

public class WanhoPageProcessor implements PageProcessor
{

    private Site site = Site
            .me()
            .setTimeOut(10000)
            .setRetryTimes(3)
            .setSleepTime(1000)
            .setCharset("UTF-8");

    @Override
    public Site getSite()
    {
        // TODO Auto-generated method stub
        return site;
    }

    @Override
    public void process(Page page)
    {
        // TODO Auto-generated method stub
        //获取当前页的所有喜报
         List<String> list = page.getHtml().xpath("//div[@class='main_l']/ul/li").all();
        //要保存喜报的集合
        Vector<ArticleVo> voLst = new Vector<>();
      //遍历喜报
        String title;
        String content;
        String img;
        for (String item : list) 
        {
            Html tmp = Html.create(item);
            //标题
            title = tmp.xpath("//div[@class='content']/h4/a/text()").toString();
            //内容
            content = tmp.xpath("//div[@class='content']/p/text()").toString();
            //图片路径
            img = tmp.xpath("//a/img/@src").toString();
            //加入集合
            ArticleVo vo = new ArticleVo(title, content, img);
            voLst.add(vo);
        }
      //保存数据至page中，后续进行持久化
        page.putField("e_list", voLst);
      //加载其它页
        page.addTargetRequests( getOtherUrls());
    }
    
    
    //其它页
    public List<String> getOtherUrls()
    {
         List<String> urlLsts = new ArrayList<>();
         for(int i=2;i<7;i++){
             urlLsts.add("http://www.wanho.net/a/jyxb/list_15_"+i+".html");
         }
        return urlLsts;
    }

}

package com.mieba.spider;

import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintWriter;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.util.Vector;

import us.codecraft.webmagic.ResultItems;
import us.codecraft.webmagic.Task;
import us.codecraft.webmagic.pipeline.Pipeline;

public class WanhoPipeline implements Pipeline
{

    @Override
    public void process(ResultItems resultItems, Task arg1)
    {
        // TODO Auto-generated method stub
        // 获取抓取过程中保存的数据
        Vector<ArticleVo> voLst = resultItems.get("e_list");
        // 持久到文件中
        PrintWriter pw = null;
        try
        {
            pw = new PrintWriter(new FileWriter("wanho.txt", true));
            for (ArticleVo vo : voLst)
            {
                pw.println(vo);
                pw.flush();
                saveImg(vo.getImg());
            }
        } catch (FileNotFoundException e)
        {
            e.printStackTrace();
        } catch (IOException e)
        {
            e.printStackTrace();
        } finally
        {
            pw.close();
        }
    }

    private void saveImg(String img)
    {
        // TODO Auto-generated method stub
        String imgUrl = "http://www.wanho.net" + img;
        InputStream is = null;
        BufferedInputStream bis = null;
        BufferedOutputStream bos = null;
        try
        {
            URL url = new URL(imgUrl);
            URLConnection uc = url.openConnection();
            is = uc.getInputStream();
            bis = new BufferedInputStream(is);
            File photoFile = new File("photo");
            if (!photoFile.exists())
            {
                photoFile.mkdirs();
            }
            String imgName = img.substring(img.lastIndexOf("/") + 1);
            File saveFile = new File(photoFile, imgName);
            bos = new BufferedOutputStream(new FileOutputStream(saveFile));
            byte[] bs = new byte[1024];
            int len;
            while ((len = bis.read(bs)) != -1)
            {
                bos.write(bs, 0, len);
            }

        } catch (MalformedURLException e)
        {
            // TODO: handle exception
            e.printStackTrace();
        } catch (IOException e)
        {
            e.printStackTrace();
        } finally
        {
            try
            {
                bos.close();
            } catch (IOException e)
            {
                e.printStackTrace();
            }
            try
            {
                bis.close();
            } catch (IOException e)
            {
                e.printStackTrace();
            }
            try
            {
                is.close();
            } catch (IOException e)
            {
                e.printStackTrace();
            }

        }
    }

}

package com.mieba.spider;

public class ArticleVo
{
    private String title;
    private String content;
    private String img;
    public String getTitle()
    {
        return title;
    }
    public void setTitle(String title)
    {
        this.title = title;
    }
    public String getContent()
    {
        return content;
    }
    public void setContent(String content)
    {
        this.content = content;
    }
    public String getImg()
    {
        return img;
    }
    public void setImg(String img)
    {
        this.img = img;
    }
    public ArticleVo(String title, String content, String img)
    {
        super();
        this.title = title;
        this.content = content;
        this.img = img;
    }
    @Override
    public String toString()
    {
        return "ArticleVo [title=" + title + ", content=" + content + ", img=" + img + "]";
    }
    
    
}

package com.mieba.spider;

import us.codecraft.webmagic.Spider;

public class Demo
{
    public static void main(String[] args)
    { // 爬取开始
        Spider 
        // 爬取过程 
        .create(new WanhoPageProcessor()) 
        // 爬取结果保存
        .addPipeline(new WanhoPipeline())
        // 爬取的第一个页面
        .addUrl("http://www.wanho.net/a/jyxb/") 
        // 启用的线程数
        .thread(5).run();
        }
    }


        

爬取到的照片

 爬取到的简报

 大家如果要使用代码，配置webmagic的依赖包即可使用。