java实现爬虫功能

/**
 * 爬取新闻信息，封装成实体bean
 */
public class GetNews {
 public List<News> getNews() {
  // 存储新闻对象
  List<News> list = new ArrayList<News>();
  try {
   // 请求DOM文档
   Document document = Jsoup.connect("http://baijia.baidu.com/").get();
   // 解析
   String selector = "h3>a";
   Elements titlels = document.select(selector);

   for (Element title : titlels) {
    // System.out.println("标题---" + title.text());
    // 再次请求a标签，获取内容
    String url = title.absUrl("href");
    Document document1 = Jsoup.connect(url).get();
    String selectTime = document1.select("span[class=time]").text();
    // System.out.println("时间---" + selectTime);
    String selectBody = document1.select(
      "div[class=article-detail]").text();
    // System.out.println("正文---" + selectBody);
    // 构成news对象加入list集合
    News news = new News();
    news.setTitle(title.text());
    news.setBody(selectBody);
    news.setDate(selectTime);
    list.add(news);
   }

  } catch (IOException e) {
   // TODO Auto-generated catch block
   e.printStackTrace();
  }

  return list;
 }

}

/*
  * 把获得的news对象存入数据库
  */
 public int save(List<News> list) {

  // sql前缀
  String sql = "insert into news (title,body,date) values";
  /*
   * 这种方式插入数据库 速度最快
   */
  for (News news : list) {
   sql = sql + "('" + news.getTitle() + "','" + news.getBody() + "','"
     + news.getDate() + "'),";
  }
  sql = sql.substring(0, sql.length() - 1);
  System.out.println(sql);
  int rows = BaseDao.executeUpdate(sql);
  return rows;
 }

/**
 * 连接数据库 通用的 工具类
 *
 */
public class BaseDao {
 // 创建需要得到JDBC API
 protected static Connection connection = null;
 protected static PreparedStatement ps = null;
 protected static ResultSet rs = null;

 // 01.获取数据库连接
 public static boolean getConnection() {
  /**
   * 获取数据库连接的4要素 连接数据库的前提
   */
  String driver = ConfigManager.getInstance().getValue("jdbc.driver");
  String url = ConfigManager.getInstance().getValue("jdbc.url");
  String userName = ConfigManager.getInstance().getValue("jdbc.userName");
  String password = ConfigManager.getInstance().getValue("jdbc.password");

  try {
   Class.forName(driver); // 加载驱动
   connection = DriverManager.getConnection(url, userName, password);
  } catch (ClassNotFoundException e) {
   e.printStackTrace();
   return false;
  } catch (SQLException e) {
   e.printStackTrace();
   return false;
  }
  return true;
 }

 /**
  * 03.增删改 executeUpdate() 返回int 代表影响数据库中的行数 delete from user; delete from
  * user where id=? and name=?;
  */
 public static int executeUpdate(String sql, Object... params) {
  int rowNum = 0;
  if (getConnection()) { // 操作数据库 肯定现有连接
   try {
    ps = connection.prepareStatement(sql);
    // 循环给sql语句中的？占位符 赋值
    for (int i = 0; i < params.length; i++) {
     ps.setObject(i + 1, params[i]);
    }
    // 执行sql语句
    rowNum = ps.executeUpdate();
   } catch (SQLException e) {
    e.printStackTrace();
   } finally {
    closeConnection(); // 关闭连接
   }

  }

  return rowNum;
 }

 /**
  * 04.查询 executeQuery() 返回ResultSet select * from user; select * from user
  * where id=? and name=?;
  */
 public static ResultSet executeQuery(String sql, Object... params) {
  if (getConnection()) { // 操作数据库 肯定现有连接
   try {
    ps = connection.prepareStatement(sql);
    // 循环给sql语句中的？占位符 赋值
    for (int i = 0; i < params.length; i++) {
     ps.setObject(i + 1, params[i]);
    }
    // 执行sql语句
    rs = ps.executeQuery();
   } catch (SQLException e) {
    e.printStackTrace();
   }
  }
  return rs;
 }

 // 02.释放资源
 public static boolean closeConnection() {
  // 如果对象都没有创建 ？ 能关闭吗？ 必须进行非空判断
  if (rs != null) {
   try {
    rs.close();
   } catch (SQLException e) {
    e.printStackTrace();
    return false;
   }
  }
  if (ps != null) {
   try {
    ps.close();
   } catch (SQLException e) {
    e.printStackTrace();
    return false;
   }
  }
  if (connection != null) {
   try {
    connection.close();
   } catch (SQLException e) {
    e.printStackTrace();
    return false;
   }
  }
  return true;
 }

}

/*
  * 输入关键字 查询 模糊查询
  */
 public List<News> selectNews(String name) {
  List<News> list = new ArrayList<News>();
  String sql = "select * from news where title like ?";
  Object[] params = { "%" + name + "%" };
  ResultSet rs = BaseDao.executeQuery(sql, params);
  try {
   // 遍历结果集
   while (rs.next()) {
    // 创建新闻对象
    News news = new News();
    // 获取每一行的每一列
    news.setId(rs.getInt("id"));
    news.setTitle(rs.getString("title"));
    news.setBody(rs.getString("body"));
    news.setDate(rs.getString("date"));
    list.add(news);
   }
  } catch (Exception e) {
   // TODO: handle exception
  }
  return list;
 }

/*
 * 单例 读取配置文件的工具类
 * */

public class ConfigManager {

 // 01.创建自身的静态对象
 private static ConfigManager manager = new ConfigManager();
 private static Properties properties;

 // 02.私有化构造
 private ConfigManager() {
  // 获取配置文件的路径
  String path = "jdbc.properties";
  properties = new Properties();
  // 创建输入流
  InputStream stream = ConfigManager.class.getClassLoader()
    .getResourceAsStream(path);
  try {
   properties.load(stream);
  } catch (IOException e) {
   e.printStackTrace();
  } finally {
   try {
    stream.close();
   } catch (IOException e) {
    e.printStackTrace();
   }
  }

 }

 // 03.提供供外部访问的接口
 public static synchronized ConfigManager getInstance() {
  return manager;
 }

 // 提供一个 根据key取得value的方法
 public static String getValue(String key) {
  return properties.getProperty(key);
 }

}

/*

*properties文件

*/

jdbc.url=jdbc:mysql://localhost:3306/test
jdbc.userName=hhr
jdbc.password=hhr
jdbc.driver=com.mysql.jdbc.Driver