我自己随便写的爬虫

有关爬虫，自己半年前写了一个，写过就忘记了，今天才看到

import java.io.IOException;
import java.net.URISyntaxException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.commons.lang.time.DateFormatUtils;
import org.apache.http.HttpEntity;
import org.apache.http.ParseException;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.util.EntityUtils;
import org.apache.xerces.util.URI;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.quartz.Job;
import org.quartz.JobExecutionContext;
import org.quartz.JobExecutionException;

/*
 * @说明
 * 由于针对网页取数据，1页有100条数据，二期数据一直是变化的；
 * 目前设计是每5分钟抓取一次，所以抓一次，存一次，之前的数据仍旧保留，但是只抓第一页数据。
 * 其他排名靠后的数据就不抓了
 * 默认排序为 totalvolpct 总成交占比
 * 
 * 
 *                     <th>名称</th>
                    <th><a href="#" onclick="return window['sortTable']('symbol', '　');">代码</a>　</th>
<th><a href="#" onclick="return window['sortTable']('totalvol', '　');">总成交量(万股)</a>　</th>
<td><a href="#" onclick="return window['sortTable']('totalvolpct', '　');">总成交量占比</a>　</td>  ----百分比，入库时是去掉百分号入库的
<td><a href="#" onclick="return window['sortTable']('totalamt', '　');">总成交额(万元)</a>　</td>
<td><a href="#" onclick="return window['sortTable']('totalamtpct', '　');">总成交额占比</a>　</td>----百分比，入库时是去掉百分号入库的
<td><a href="#" onclick="return window['sortTable']('avgprice', '　');">平均成交价(元)</a>　</td>
<td><a href="#" onclick="return window['sortTable']('kuvolume', '↓');">主买量(万股)</a>↓</td>
<td><a href="#" onclick="return window['sortTable']('kevolume', '　');">中性量(万股)</a>　</td>
<td><a href="#" onclick="return window['sortTable']('kdvolume', '　');">主卖量(万股)</a>　</td>
                    <th>详情　</th>


建数据库表时，字段顺序一定要按照上面的顺序来建，否则会有问题

 * 
 * 
*/


public class Crawler implements Job{

    
    private String url="http://vip.stock.finance.sina.com.cn/quotes_service/view/cn_bill_sum.php?num=100&sort=totalvolpct&asc=0&volume=40000&type=0&dpc=1&page=1";
    private String encode="UTF-8";
    
    public String getUrlData() {

        String out=new String();
        
        //---大单分析---
        //---http://vip.stock.finance.sina.com.cn/quotes_service/view/cn_bill_sum.php?num=100&sort=totalvolpct&asc=0&volume=40000&type=0&dpc=1&page=2

        // 获得Http客户端(可以理解为:你得先有一个浏览器;注意:实际上HttpClient与浏览器是不一样的)
                CloseableHttpClient httpClient = HttpClientBuilder.create().build();
         
//                // 创建Post请求

                HttpPost httpPost = new  HttpPost(url);
         
                //---下面这句话暂时没有起作用，不知道原因；其实参数是可以不用放在上面的httpPost对象中的
                StringEntity entity = new StringEntity("num=100&sort=totalvolpct&asc=0&volume=40000&type=0&dpc=1&page=1", encode);
         
                // post请求是将参数放在请求体里面传过去的;这里将entity放入post请求体中
                httpPost.setEntity(entity);
         
                httpPost.setHeader("Content-Type", "text/html;charset=utf8");
         
                // 响应模型
                CloseableHttpResponse response = null;
                try {
                    // 由客户端执行(发送)Post请求
                    response = httpClient.execute(httpPost);
                    // 从响应模型中获取响应实体
                    HttpEntity responseEntity = response.getEntity();
         
                    System.out.println("响应状态为:" + response.getStatusLine());
                    if (responseEntity != null) {
                        System.out.println("响应内容长度为:" + responseEntity.getContentLength());
                        //System.out.println("响应内容为:" + EntityUtils.toString(responseEntity,"GBK"));
                        out=EntityUtils.toString(responseEntity,"GBK");
                    }
                } catch (ClientProtocolException e) {
                    e.printStackTrace();
                } catch (ParseException e) {
                    e.printStackTrace();
                } catch (IOException e) {
                    e.printStackTrace();
                } finally {
                    try {
                        // 释放资源
                        if (httpClient != null) {
                            httpClient.close();
                        }
                        if (response != null) {
                            response.close();
                        }
                    } catch (IOException e) {
                        e.printStackTrace();
                    }
                }
        
        
        
        return out;

    }

    
    //---想直接跳过字符串处理，暂时没成功---//
    public void DealUrlString(String inStr,org.springframework.jdbc.core.JdbcTemplate db) {
        
        String out=new String();
        
        Document doc = Jsoup.parseBodyFragment(inStr);
        
        Element et = doc.getElementById("divListTemplate");
        Elements et_tab = et.getElementsByTag("table");
    
        Elements trs = et_tab.first().getElementsByTag("tr");
        
        
        System.out.println("====size===="+trs.size());  
        
        
        int n=0;
        
        
        
        for (Element element : trs) {
            if(n==0) {
                n++;
            }
            else {
                StringBuffer insert_sql=new StringBuffer();
                
                insert_sql.append("insert into stock_bigdeal_analyse (cn_name,symbol,totalvol,totalvolpct,totalamt,totalamtpct,avgprice,kuvolume,kevolume,kdvolume,input_time) values ( ");
                
                Elements ele_ths= element.getElementsByTag("th");
                insert_sql.append(  "'" + ele_ths.get(0).text().trim()+"', ");
                insert_sql.append(  "'" + ele_ths.get(1).text().replaceAll("　", "")+"', ");//--看不见的特殊符号，
                
                
                Elements ele_tds= element.getElementsByTag("td");
              
                insert_sql.append( "'" + ele_tds.get(0).text().trim().replaceAll("　", "").replaceAll(",", "")+"', ");
                insert_sql.append(  "'" + ele_tds.get(1).text().trim().replaceAll("　", "").replaceAll("%", "")+"',");
                insert_sql.append(  "'" + ele_tds.get(2).text().trim().replaceAll("　", "").replaceAll(",", "")+"',");
                insert_sql.append(  "'" + ele_tds.get(3).text().trim().replaceAll("　", "").replaceAll("%", "")+"',");
                insert_sql.append(  "'" + ele_tds.get(4).text().trim().replaceAll("　", "").replaceAll(",", "")+"',");
                insert_sql.append(  "'" + ele_tds.get(5).text().trim().replaceAll("　", "").replaceAll(",", "")+"',");
                insert_sql.append(  "'" + ele_tds.get(6).text().trim().replaceAll("　", "").replaceAll(",", "")+"',");
                insert_sql.append(  "'" + ele_tds.get(7).text().trim().replaceAll("　", "").replaceAll(",", "")+"',");
                
                insert_sql.append(  "'" +DateFormatUtils.format(new Date(),"yyyyMMddHHmmssSSS")+"') ");
                
              //  System.out.println(insert_sql);
                
                db.execute(insert_sql.toString());
                
            }
        }
        
        
        
    }
    
    
    
    public  static void main(String args[]) {
        
        Crawler cr= new Crawler();
        
        SpringDb sd = new SpringDb();
        org.springframework.jdbc.core.JdbcTemplate db= sd.getJdbc();
        
        String data_in = cr.getUrlData();
        cr.DealUrlString(data_in,db);
        
        
    }


    @Override
    public void execute(JobExecutionContext arg0) throws JobExecutionException {

        Crawler cr= new Crawler();
        
        SpringDb sd = new SpringDb();
        org.springframework.jdbc.core.JdbcTemplate db= sd.getJdbc();
        
        String data_in = cr.getUrlData();
        cr.DealUrlString(data_in,db);
        
    }
    
    
    
    
    
}