HttpClient+Jsoup爬取页面数据

为什么不使用爬虫框架？

　　原本使用的WebMagic框架，但是报了协议版本不一致异常，百度很多方法没解决掉，而且也是自己写着玩，就换了方式； 

javax.net.ssl.SSLException: Received fatal alert: protocol_version

jar包依赖：

        <!--解析网页数据-->
        <dependency>
            <groupId>org.jsoup</groupId>
            <artifactId>jsoup</artifactId>
            <version>1.10.2</version>
        </dependency>

        <dependency>
            <groupId>org.apache.httpcomponents</groupId>
            <artifactId>httpclient</artifactId>
            <version>4.5.2</version>
        </dependency>

代码：

import com.maxinhai.world.utils.JdbcUtils;
import org.apache.http.HttpEntity;
import org.apache.http.StatusLine;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.config.RegistryBuilder;
import org.apache.http.conn.socket.ConnectionSocketFactory;
import org.apache.http.conn.socket.PlainConnectionSocketFactory;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.ssl.SSLContexts;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import javax.net.ssl.SSLContext;
import java.io.IOException;
import java.sql.*;
import java.sql.Date;
import java.util.*;

/**
 * @program: world
 * @description: 雅拉伊图片爬虫
 * @author: XinHai.Ma
 * @create: 2021-10-21 20:33
 */
public class YaLaYiImageReptile {

    // 解决问题: javax.net.ssl.SSLException: Received fatal alert: protocol_version
    private static final PoolingHttpClientConnectionManager HTTP_CLIENT_CONNECTION_MANAGER;
    private static final CloseableHttpClient HTTP_CLIENT;
    static {
        SSLContext ctx = SSLContexts.createSystemDefault();
        SSLConnectionSocketFactory fac =
                new SSLConnectionSocketFactory(ctx, new String[]{"SSLv2Hello", "TLSv1.2"}, null, SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);

        HTTP_CLIENT_CONNECTION_MANAGER = new PoolingHttpClientConnectionManager(RegistryBuilder
                .<ConnectionSocketFactory> create().register("http", PlainConnectionSocketFactory.getSocketFactory())
                .register("https", fac).build());
        HTTP_CLIENT_CONNECTION_MANAGER.setDefaultMaxPerRoute(100);
        HTTP_CLIENT_CONNECTION_MANAGER.setMaxTotal(200);
        RequestConfig requestConfig = RequestConfig.custom().setConnectionRequestTimeout(60000).setConnectTimeout(60000)
                .setSocketTimeout(60000).build();

        HTTP_CLIENT = HttpClientBuilder.create().setConnectionManager(HTTP_CLIENT_CONNECTION_MANAGER)
                .setDefaultRequestConfig(requestConfig).build();
    }

    /**
     * 报异常 => javax.net.ssl.SSLException: Received fatal alert: protocol_version
     * @param args
     */
    public static void main(String[] args) {
        // 要爬取的页面
        for(int i=1; i<=10; i++) {
            if(i != 1) {
                reptile("https://www.yalayi.com/gallery/index_" + i + ".html");
            } else {
                reptile("https://www.yalayi.com/gallery/");
            }
        }
    }

    private static void reptile(String url) {
        try {
            HttpGet httpGet = new HttpGet(url);
            CloseableHttpResponse httpResponse = HTTP_CLIENT.execute(httpGet);
            StatusLine statusLine = httpResponse.getStatusLine();
            if(statusLine.getStatusCode() == 200) {
                HttpEntity httpEntity = httpResponse.getEntity();
                String result = EntityUtils.toString(httpEntity, "utf-8");
                //System.out.println(statusLine + "=>" + result);

                Document document = Jsoup.parse(result);
                Elements urlElements = document.select("body > div.main.bgf6 > div.gallery-list.list-box > div > ul > li > div.img-box > a");
                Elements imageElements = document.select("body > div.main.bgf6 > div.gallery-list.list-box > div > ul > li > div.img-box > a > img");
                Elements titleElements = document.select("body > div.main.bgf6 > div.gallery-list.list-box > div > ul > li > div.text-box > p > a");
                Elements sizeElements = document.select("body > div.main.bgf6 > div.gallery-list.list-box > div > ul > li > div.img-box > em");
                List<Map<String, String>> dataList = new ArrayList<>();
                for (int i = 0; i < titleElements.size(); i++) {
                    Element urlElement = urlElements.get(i);
                    Element imageElement = imageElements.get(i);
                    Element titleElement = titleElements.get(i);
                    Element sizeElement = sizeElements.get(i);
                    String href = urlElement.attr("href");
                    String src = imageElement.attr("src");
                    String title = titleElement.text();
                    String size = sizeElement.text();
                    System.out.println("标题:" + title + " 封面:" + src + " 画质:" + size + " 详情页:" + href);

                    Map<String, String> data = new HashMap<>();
                    data.put("pageUrl", href);
                    data.put("imageUrl", src);
                    data.put("title", title);
                    data.put("size", size);
                    dataList.add(data);
                }
                saveDate(dataList);
            }

        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    /**
     * 保存爬取数据
     * @param dataList
     */
    private static void saveDate(List<Map<String, String>> dataList) {
        List<String> titleList = new ArrayList<>();
        for (int i = 0; i < dataList.size(); i++) {
            Map<String, String> data = dataList.get(i);
            String title = data.get("title");
            titleList.add(title);
        }

        // 查询已经爬取的数据，做筛选处理
        List<String> resultList = query(titleList);
        Iterator<Map<String, String>> iterator = dataList.iterator();
        while (iterator.hasNext()) {
            Map<String, String> next = iterator.next();
            if(resultList.contains(next.get("title"))) {
                iterator.remove();
            }
        }

        // 获取数据库连接对象
        Connection connection = JdbcUtils.getConnection();

        // 插入数据
        String insert_novel = "INSERT INTO `world`.`yalayi_image_reptile`(`is_active`, `is_delete`, `create_time`, `title`, `size`, `cover_url`, `page_url`) " +
                "VALUES (?, ?, ?, ?, ?, ?, ?)";

        try {
            PreparedStatement preparedStatement = connection.prepareStatement(insert_novel);
            for (int i = 0; i < dataList.size(); i++) {
                Map<String, String> data = dataList.get(i);
                preparedStatement.setInt(1, 0);
                preparedStatement.setInt(2, 0);
                preparedStatement.setDate(3, new Date(System.currentTimeMillis()));
                preparedStatement.setString(4, data.get("title"));
                preparedStatement.setString(5, data.get("size"));
                preparedStatement.setString(6, data.get("imageUrl"));
                preparedStatement.setString(7, data.get("pageUrl"));
                preparedStatement.addBatch();
            }

            int[] row = preparedStatement.executeBatch();
            System.out.println("插入" + row.length + "行");
        } catch (SQLException e) {
            e.printStackTrace();
        }
    }


    /**
     * in范围查询
     * @param titleList
     * @return
     */
    private static List<String> query(List<String> titleList) {
        List<String> titles = new ArrayList<>();
        String sql = "SELECT title FROM `yalayi_image_reptile` where is_delete=0 and title in (?)";
        String stringFromList = getStringFromList(titleList);
        String formatSql = String.format(sql.replace("?", "%s"), stringFromList);
        try {
            Connection connection = JdbcUtils.getConnection();
            PreparedStatement preparedStatement = connection.prepareStatement(formatSql);
            ResultSet resultSet = preparedStatement.executeQuery();
            while (resultSet.next()) {
                String title = resultSet.getString(1);
                titles.add(title);
            }
        } catch (SQLException e) {
            e.printStackTrace();
        }
        return titles;
    }

    /**
     * 格式化参数
     * @param paramList
     * @return
     */
    private static String getStringFromList(List<String> paramList) {
        StringBuffer sb = new StringBuffer();
        for (int i = 0; i < paramList.size(); i++) {
            if(i == paramList.size()-1) {
                sb.append("'" + paramList.get(i) + "'");
            } else {
                sb.append("'" + paramList.get(i) + "'");
                sb.append(",");
            }
        }
        return sb.toString();
    }

}

Ps: 1. 上面代码中没有对数据库连接做关闭处理，是因为其他代码会用到这个连接对象，就没有关闭；

　　2. document.select()方法里的css不会写没关系，在页面中找到你要爬取的dom节点，右键检查，就能看到html代码，选中要爬取的html标签，右键复制为selector即可；

关于协议版本不一致问题：

　　WebMagic默认的HttpClient只会用TLSv1去请求，对于某些只支持TLS1.2的站点（例如https://juejin.im/） ，就会报错；

　　处理方法：https://github.com/code4craft/webmagic/issues/701；