爬虫效果--详细

项目简介：
此项目为了方便同学们进行爬虫编写。涉及到的知识点有Html，Javascript、JQuery
Echarts，Servlet 3.0,Jsoup，FastJson、IO流
项目概要设计：
1、包结构的设计 必须满足MVC开发模式---解耦

package com.hyxy.food.entity;
public class Food {
    private int id;//区别不同的商店
    private String name;//名称
    private int num;//评论条数
    public int getId() {
        return id;
    }
    public void setId(int id) {
        this.id = id;
    }
    public String getName() {
        return name;
    }
    public void setName(String name) {
        this.name = name;
    }
    public int getNum() {
        return num;
    }
    public void setNum(int num) {
        this.num = num;
    }
    public Food(int id, String name, int num) {
        super();
        this.id = id;
        this.name = name;
        this.num = num;
    }
    public Food() {
        super();
    }    
}

package com.hyxy.food.util;

import java.io.IOException;
import java.io.InputStream;
import java.sql.Connection;
import java.sql.DriverManager;
import java.util.Properties;
public class ConnectMysqlDB {
    public static Connection getConnection() {
        Connection conn=null;
        Properties p=new Properties();
        InputStream in=ConnectMysqlDB.class.getClassLoader().getResourceAsStream("jdbc.properties");
        try {
        p.load(in);
            String url=p.getProperty("url");
            String user=p.getProperty("user");
            String password=p.getProperty("pwd");
            String driver=p.getProperty("driver");
            Class.forName(driver);
            conn=DriverManager.getConnection(url, user, password);
        } catch (Exception e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }

        return conn;
    }
    public static void main(String[] args) {
        getConnection();
    }
}

jdbc.properties  --->src根目录下
driver=com.mysql.jdbc.Driver
url=jdbc:mysql://192.168.0.199:3306/spider?useUnicode=true&characterEncoding=utf-8
user=1805
pwd=123456

package com.hyxy.food.dao;
import java.sql.Connection;
import com.hyxy.food.entity.Food;
import com.hyxy.food.util.ConnectMysqlDB;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;

public class FoodDao {
    private  Connection conn;
    public FoodDao() {
       if(conn==null) {
        conn=ConnectMysqlDB.getConnection();
    }
     }
    public boolean addFood(Food f) {
        String sql="insert into food(name,num) value(?,?)";
        PreparedStatement ps;
        boolean flag=false;
        try {
            ps = conn.prepareStatement(sql);
            ps.setString(1, f.getName());
            ps.setInt(2, f.getNum());
            ps.executeUpdate();
            flag=true;
        } catch (SQLException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
        return flag;
    }
    public List<Food> list(){
        String sql="select name,num from food";
        List<Food> list=new ArrayList<Food>();
        try {
            PreparedStatement ps=conn.prepareStatement(sql);
            ResultSet rs= ps.executeQuery();
            while(rs.next()) {
                Food f=new Food();
                f.setName(rs.getString("name"));
                f.setNum(rs.getInt("num"));
                list.add(f);
            }
        } catch (SQLException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
        return list;
    }
}

package com.hyxy.food.controller;

import java.io.IOException;
import javax.servlet.ServletException;
import javax.servlet.annotation.WebServlet;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.hyxy.food.dao.FoodDao;
import com.hyxy.food.entity.Food;
@WebServlet("/food")
public class FoodServlet extends HttpServlet {
    private static final long serialVersionUID = 1L;
    public FoodServlet() {
        super();
        // TODO Auto-generated constructor stub
    }
    protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
        doPost(request, response);
    }
    protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
        String url=request.getParameter("url");//http://www.mafengwo.cn/cy/10035/0-0-0-0-0-
        int sum=Integer.parseInt(request.getParameter("sum"));
        //爬虫开始
        for (int i = 1; i <= sum; i++) {
            System.out.println("爬取第"+i+"页数据");
            //爬虫第一步,获取一个爬虫的Document对象
            Document d=Jsoup.connect(url+i+".html").get();
            //输出Document对象
    //        System.out.println(d.html());
            //第二步观察网页具体信息，爬取想要的信息
            Elements es= d.select("li[class=item clearfix]");
            boolean flag=false;
            for (Element element : es) {
                String title=element.select("div[class=title]").select("h3").select("a").first().text();
                int num=Integer.parseInt(element.select("div[class=grade]").select("p[class=rev-num]").select("em").text());
                Food food=new Food();
                food.setName(title);
                food.setNum(num);
                FoodDao dao=new FoodDao();
                flag = dao.addFood(food);
            }
            if(flag) {
                System.out.println("success！！！");
            }else {
                System.out.println("false!!!");
            }
//            request.getRequestDispatcher("list").forward(request, response);
        }
    }

}

package com.hyxy.food.test;

import java.io.IOException;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

public class FoodTest {
    public static void main(String[] args) throws IOException {
        int sum=20;
        for (int i = 1; i <= sum; i++) {
            System.out.println("爬取第"+i+"页数据");
            String url="http://www.mafengwo.cn/cy/10035/0-0-0-0-0-"+i+".html";
            //爬虫第一步,获取一个爬虫的Document对象
            Document d=Jsoup.connect(url).get();
            //输出Document对象
    //        System.out.println(d.html());
            //第二步观察网页具体信息，爬取想要的信息
            Elements es= d.select("li[class=item clearfix]");
            for (Element element : es) {
                String title=element.select("div[class=title]").select("h3").select("a").first().text();
                int num=Integer.parseInt(element.select("div[class=grade]").select("p[class=rev-num]").select("em").text());
                System.out.println(title+":"+num);
            }
        }            
    }
}