JAVA爬虫入门学习

之前分享一个简单的获取数据流的方法，

今天分享下学习到的SpringBoot+Maven的框架

首先pom.xml

<!-- 继承父包 -->
    <parent>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-parent</artifactId>
        <version>1.5.9.RELEASE</version>
    </parent>

    <dependencies>
        <!-- spring-boot使用jetty容器配置begin -->
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-web</artifactId>
            <!-- 排除默认的tomcat,引入jetty容器. -->
            <exclusions>
                <exclusion>
                    <groupId>org.springframework.boot</groupId>
                    <artifactId>spring-boot-starter-tomcat</artifactId>
                </exclusion>
            </exclusions>
        </dependency>
        <!-- jetty 容器. -->
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-jetty</artifactId>
        </dependency>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-test</artifactId>
            <scope>test</scope>
        </dependency>
        <!-- spring-boot使用jetty容器配置end -->

        <dependency>
            <groupId>org.jsoup</groupId>
            <artifactId>jsoup</artifactId>
            <version>1.10.3</version>
        </dependency>

    </dependencies>

其中Jetty 是一个开源的servlet容器，它为基于Java的web容器，例如JSP和servlet提供运行环境。
Jetty是使用JAVA编写的，它的API以一组JAR包的形式发布。
开发人员可以将Jetty容器实例化成一个对象，可以迅速为一些独立运行（stand-alone）的Java应用提供网络和web连接。

Jetty提供了一下HTTPClient的一些发包方法。
下面是控制器内写的接口

@Controller  /*返回视图页名称*/
//@RestController  返回json数据类型
public class MainController {

    @Autowired//根据类型匹配需要配合Qualifier
    //@Qualifier//名称
    //@Resource//根据名称自动匹配，区别于@Autowired
    public GetCaptcha Captcha;

    @GetMapping("Captcha")//等同于下面
    //@RequestMapping(value = "name",method = RequestMethod.GET)
    public @ResponseBody String index(HttpServletRequest request){//错误需要另外处理

        //初始化
        HttpClient httpClient = Captcha.GetHttpClient();
        CaptchaData captchaData = new CaptchaData();
        captchaData.setHttpClient(httpClient);

        if(httpClient==null){
            return "创建对象错误，检查代理是否到期";
        }

        CaptchaData imgValue = Captcha.GetCaptcha(captchaData);

        request.getSession().setAttribute("captchaData",imgValue);

        String htmlText = imgValue.getImg()+"<form action="/json" method="POST">
" +
                "请输入用户名:<br>
" +
                "<input type="text" name="name" value="张三">
" +
                "<br>
" +
                "身份证号:<br>
" +
                "<input type="text" name="cardId" value="123456789098765432">
" +
                "<br>
" +
                "验证码:<br>
" +
                "<input type="text" name="captcha" value="">
" +
                "<br><br>
" +
                "<input type="submit" value="提交">
" +
                "</form> ";
        return htmlText;//返回视图名称
    }

    @PostMapping("json")
    public @ResponseBody String resp(Userdb user,HttpServletRequest request){

        CaptchaData captchaData = (CaptchaData) request.getSession().getAttribute("captchaData");
        HttpClient httpClient = captchaData.getHttpClient();

        String returnValue = "";
        try {
            returnValue = Captcha.PostData(user,captchaData);
        }catch (Exception e){
            returnValue = "数据传输出现错误，请检查数据信息";
        }finally {
            try{
                httpClient.stop();
            }
            catch (Exception e){
                returnValue += " 
 HttpClient关闭异常";
            }
        }
        return returnValue;
    }
}

在控制器中调用的方法分别包括：

获取验证码、发送数据到目标地址、初始化httpClient对象

代码如下：

public CaptchaData GetCaptcha(CaptchaData captchaData) {
        HttpClient httpClient = GetHttpClient();
        try {
            ContentResponse response1 = httpClient.newRequest("https://www.baidu.com").send();
            //System.out.println(response1.getContentAsString());
            //首页
            String TOKEN;
            response1 = httpClient.newRequest("XXXXXXXX/index1.do")
                    .method(HttpMethod.GET)
                    .header(HttpHeader.REFERER, "https://XXXXXXXX/")
                    .send();

            //点击进入页面
            response1 = httpClient.newRequest("XXXXXXXX/xxxxx.xxx?xxxx=initLogin")
                    .method(HttpMethod.GET)
                    .header(HttpHeader.REFERER, "https://XXXXXXXX/index1.do")
                    .send();

            //点击注册页面
            response1 = httpClient.newRequest("https://XXXXXXXX/xxxxx?method=initReg")
                    .method(HttpMethod.POST)
                    .header("Referer", "https://XXXXXXXX/loginreg.jsp")
                    .send();

            //获取TOKEN
            Document documentUrlOne = Jsoup.parse(response1.getContentAsString());
            Elements firstNode = documentUrlOne.select("div.white-con");
            Elements formNode = firstNode.select("form[action]");
            Element inputNode = formNode.select("input").first();
            TOKEN = inputNode.attr("value");
            //存入
            captchaData.setTOKEN(TOKEN);
            //TOKEN需要在注册提交页面生成的才有效
            //System.out.println(TOKEN+"---获取org.apache.struts.taglib.html.TOKEN----");

            //获取验证码。
            String timeString = String.valueOf(new Date().getTime());
            String ImgStr = "https://XXXXXXXX/imgrc.do?a=" + timeString;

            response1 = httpClient.newRequest(ImgStr)
                    .method(HttpMethod.GET)
                    .header("Referer", "https://XXXXXXXXXXXXXXXX.do")
                    .send();
            //图片类型转换
            String encodeBase64 = Base64.getEncoder().encodeToString(response1.getContent());
            //<img src='data:image/png;base64,bsae64Str'/>
            //String filepath = "XXXXXXXXa.png";
            //convertBase64DataToImage(encodeBase64, filepath);
            //System.out.printf("图片下载成功")
            captchaData.setImg("<img src='data:image/png;base64," + encodeBase64 + "'/>");
            captchaData.setResponse1(response1);
            return captchaData;
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            return captchaData;
        }
    }

public String PostData(Userdb User, CaptchaData captchaData) {

        String Name = User.getName();
        String CardId = User.getCardId();
        String CAPTCHA = User.getCaptcha();
        //发送信息提交
        String TOKEN =captchaData.getTOKEN();
        String postdata = "org.apache.struts.taglib.html.TOKEN=" + TOKEN +
                "&method=checkIdentity" +
                "&userInfoVO.name=" + Name +
                "&userInfoVO.certType=0" +
                "&userInfoVO.certNo=" + CardId +
                "&_%40IMGRC%40_=" + CAPTCHA +
                "&1=on";
        HttpClient httpClient = captchaData.getHttpClient();


        try{
            ContentResponse response1 = httpClient.newRequest("https://xxxxxxxxxdo?" + postdata)
                    .method(HttpMethod.POST)
                    .header("Referer", "https:/xxxxxxxx.do")
                    .send();

            //  获取结果。检查是否出现错误
            Document documentPage = Jsoup.parse(response1.getContentAsString());
            Elements ErrorNode = documentPage.select("div.erro_div1");
            Elements ErrorValueNode = ErrorNode.select("span[id=_error_field_]");
            String ErrorValue = ErrorValueNode.text();
            if (ErrorValue.equals("")) {
                return "身份信息正确---END";
                //System.out.println("身份信息正确---END");
            } else {
                return ErrorValue + "cw---END";
                //System.out.println(ErrorValue+"---END");
            }
        }catch (Exception e){
            return "异常哎";
        }

    }

public HttpClient GetHttpClient() {
        //创建HTTPClient对象，
        //初始化
        SslContextFactory sslContextFactory = new SslContextFactory();
        sslContextFactory.setTrustAll(true);
        HttpClient httpClient = new HttpClient(sslContextFactory);
        httpClient.setConnectTimeout(10000);
        httpClient.setFollowRedirects(true);
        httpClient.setUserAgentField(new HttpField("User-Agent", "xxxxxxxxxxxxxxxxxxxxxxxxxx"));

        ProxyConfiguration proxyConfig = httpClient.getProxyConfiguration();
        HttpProxy proxy = new HttpProxy("xxxxxx", xxx);
        proxyConfig.getProxies().add(proxy);
        try {
            httpClient.start();
            return httpClient;
        } catch (Exception e) {
            return null;
        }

    }

//下载图片
    public void convertBase64DataToImage(String base64ImgData, String filePath) throws IOException {
        BASE64Decoder d = new BASE64Decoder();
        byte[] bs = d.decodeBuffer(base64ImgData);
        FileOutputStream os = new FileOutputStream(filePath);
        os.write(bs);
        os.close();
    }

最后是application.yml配置

server:
    port: 8080


spring:
    http:
        encoding:
        charset: utf-8
        enabled: true
        force: true
    thymeleaf:
        mode: HTML5
        encoding: UTF-8
        content-type: text/html
        cache: false
        prefix: classpath:/templates/
        suffix: .html
    resources:
        static-locations: classpath:/static/

一个完整的爬虫就实现了，使用服务接口的形式