zoukankan      html  css  js  c++  java
  • 爬虫:HttpClient Jsoup 爬虫学习

    maven工程:

     log4j.properties

    log4j.rootLogger=DEBUG,A1
    log4j.logger.org.example=DEBUG
    log4j.appender.A1=org.apache.log4j.ConsoleAppender
    log4j.appender.A1.layout=org.apache.log4j.PatternLayout
    log4j.appender.A1.layout.ConversionPattern=%-d{yyyy-MM-dd HH:mm:ss,SSS} [%t] [%c]-[%p] %m%n

    pom.xml

    <?xml version="1.0" encoding="UTF-8"?>
    <project xmlns="http://maven.apache.org/POM/4.0.0"
             xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
             xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
        <modelVersion>4.0.0</modelVersion>
    
        <groupId>org.example</groupId>
        <artifactId>com.mwq.webmagic</artifactId>
        <version>1.0-SNAPSHOT</version>
        <dependencies>
            <dependency>
                <groupId>org.apache.httpcomponents</groupId>
                <artifactId>httpclient</artifactId>
                <version>4.5.2</version>
            </dependency>
            <!-- https://mvnrepository.com/artifact/org.slf4j/slf4j-log4j12 -->
            <dependency>
                <groupId>org.slf4j</groupId>
                <artifactId>slf4j-log4j12</artifactId>
                <version>1.7.25</version>
            </dependency>
            <!-- https://mvnrepository.com/artifact/org.jsoup/jsoup -->
            <dependency>
                <groupId>org.jsoup</groupId>
                <artifactId>jsoup</artifactId>
                <version>1.10.2</version>
            </dependency>
            <!-- https://mvnrepository.com/artifact/junit/junit -->
            <dependency>
                <groupId>junit</groupId>
                <artifactId>junit</artifactId>
                <version>4.12</version>
                <scope>test</scope>
            </dependency>
            <!-- https://mvnrepository.com/artifact/commons-io/commons-io -->
            <dependency>
                <groupId>commons-io</groupId>
                <artifactId>commons-io</artifactId>
                <version>2.6</version>
            </dependency>
            <!-- https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 -->
            <dependency>
                <groupId>org.apache.commons</groupId>
                <artifactId>commons-lang3</artifactId>
                <version>3.7</version>
            </dependency>
    
    
        </dependencies>
    
    
    
    </project>

    一、HttpClient模拟浏览器请求网页

    package com.mwq.crawler.test;
    
    import org.apache.http.HttpEntity;
    import org.apache.http.client.methods.CloseableHttpResponse;
    import org.apache.http.client.methods.HttpGet;
    import org.apache.http.impl.client.CloseableHttpClient;
    import org.apache.http.impl.client.HttpClients;
    import org.apache.http.util.EntityUtils;
    
    import java.io.IOException;
    
    public class Crawler {
        public static void main(String[] args) throws Exception {
            //打开浏览器,创建hhtpclient对象
            CloseableHttpClient httpClient = HttpClients.createDefault();
    
            //输入网址,发起GET请求创建HttpGet对象
            HttpGet httpGet = new HttpGet("http://www.itcast.cn");
    
            //按回车,发起请求,返回响应,使用HttpClient对象发起请求
            CloseableHttpResponse response = httpClient.execute(httpGet);
    
            if(response.getStatusLine().getStatusCode() == 200){
                HttpEntity httpEntity = response.getEntity();
                String content = EntityUtils.toString(httpEntity,"utf-8");
    
                System.out.println(content);
            }
    
        }
    }

    二、HttpGet

    package com.mwq.crawler.test;
    
    import org.apache.http.client.methods.CloseableHttpResponse;
    import org.apache.http.client.methods.HttpGet;
    import org.apache.http.impl.client.CloseableHttpClient;
    import org.apache.http.impl.client.HttpClients;
    import org.apache.http.util.EntityUtils;
    
    import java.io.IOException;
    
    public class HttpGetTest {
        public static void main(String[] args) {
            //创建httpjclient 对象
            CloseableHttpClient httpClient = HttpClients.createDefault();
    
            //创建HttpGet对象,设置URL访问地址
            HttpGet httpGet = new HttpGet("http://www.itcast.cn");
    
            //使用HttpClient发起请求
            CloseableHttpResponse response = null;
            try {
                response = httpClient.execute(httpGet);
                if (response.getStatusLine().getStatusCode() == 200) {
                    String content = EntityUtils.toString(response.getEntity());
                    System.out.println(content.length());
                }
            } catch (IOException e) {
                e.printStackTrace();
            } finally {
                //关闭response
                try {
                    response.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
                try {
                    httpClient.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
    
        }
    }

    三、带参数的HttpGet

    package com.mwq.crawler.test;
    
    import org.apache.http.client.methods.CloseableHttpResponse;
    import org.apache.http.client.methods.HttpGet;
    import org.apache.http.client.utils.URIBuilder;
    import org.apache.http.impl.client.CloseableHttpClient;
    import org.apache.http.impl.client.HttpClients;
    import org.apache.http.util.EntityUtils;
    
    import java.io.IOException;
    import java.net.URISyntaxException;
    
    public class HttpGetParamTest {
        public static void main(String[] args) throws URISyntaxException {
            //创建httpjclient 对象
            CloseableHttpClient httpClient = HttpClients.createDefault();
    
            //设置请求地址是:http://yun.itheima.com/search?keys=Java
            //创建urlbuilder
            URIBuilder uriBuilder = new URIBuilder("http://yun.itheima.com/search");
            //设置参数
            uriBuilder.setParameter("keys","Java");
            //创建HttpGet对象,设置URL访问地址
            HttpGet httpGet = new HttpGet(uriBuilder.build());
            System.out.println("发起请求的信息:" + httpGet);
            //使用HttpClient发起请求
            CloseableHttpResponse response = null;
            try {
                response = httpClient.execute(httpGet);
                if (response.getStatusLine().getStatusCode() == 200) {
                    String content = EntityUtils.toString(response.getEntity());
                    System.out.println(content.length());
                }
            } catch (IOException e) {
                e.printStackTrace();
            } finally {
                //关闭response
                try {
                    response.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
                try {
                    httpClient.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
    
        }
    }

    四、HttpPost

    package com.mwq.crawler.test;
    
    import org.apache.http.client.methods.CloseableHttpResponse;
    import org.apache.http.client.methods.HttpGet;
    import org.apache.http.client.methods.HttpPost;
    import org.apache.http.impl.client.CloseableHttpClient;
    import org.apache.http.impl.client.HttpClients;
    import org.apache.http.util.EntityUtils;
    
    import java.io.IOException;
    
    public class HttpPostTest {
        public static void main(String[] args) {
            //创建httpjclient 对象
            CloseableHttpClient httpClient = HttpClients.createDefault();
    
            //创建HttpGet对象,设置URL访问地址
            HttpPost httpPost = new HttpPost("http://www.itcast.cn");
    
            //使用HttpClient发起请求
            CloseableHttpResponse response = null;
            try {
                response = httpClient.execute(httpPost);
                if (response.getStatusLine().getStatusCode() == 200) {
                    String content = EntityUtils.toString(response.getEntity());
                    System.out.println(content.length());
                }
            } catch (IOException e) {
                e.printStackTrace();
            } finally {
                //关闭response
                try {
                    response.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
                try {
                    httpClient.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
    
        }
    }

    五、带参数的HttpPost

    package com.mwq.crawler.test;
    
    import org.apache.http.NameValuePair;
    import org.apache.http.client.entity.UrlEncodedFormEntity;
    import org.apache.http.client.methods.CloseableHttpResponse;
    import org.apache.http.client.methods.HttpGet;
    import org.apache.http.client.methods.HttpPost;
    import org.apache.http.client.utils.URIBuilder;
    import org.apache.http.impl.client.CloseableHttpClient;
    import org.apache.http.impl.client.HttpClients;
    import org.apache.http.message.BasicNameValuePair;
    import org.apache.http.util.EntityUtils;
    
    import java.io.IOException;
    import java.io.UnsupportedEncodingException;
    import java.net.URISyntaxException;
    import java.util.ArrayList;
    import java.util.List;
    
    public class HttpPostParamTest {
        public static void main(String[] args) throws URISyntaxException, Exception {
            //创建httpjclient 对象
            CloseableHttpClient httpClient = HttpClients.createDefault();
    
            //创建HttpGet对象,设置URL访问地址
            HttpPost httpPost = new HttpPost("http://yun.itheima.com/search");
            //声明LIST稽核,封装表单中的参数
            List<NameValuePair> params  = new ArrayList<NameValuePair>();
            //设置请求地址:http://yun.itheima.com/search?keys=Java
            params.add(new BasicNameValuePair("keys","Java"));
            //设置表单的Entity 对象,第一个参数就是封装号的表单数据,第二个参数就是编码
            UrlEncodedFormEntity formEntity = new UrlEncodedFormEntity(params,"utf8");
            httpPost.setEntity(formEntity);
            System.out.println("发起请求的信息:" + httpPost);
            //使用HttpClient发起请求
            CloseableHttpResponse response = null;
            try {
                response = httpClient.execute(httpPost);
                if (response.getStatusLine().getStatusCode() == 200) {
                    String content = EntityUtils.toString(response.getEntity());
                    System.out.println(content.length());
                }
            } catch (IOException e) {
                e.printStackTrace();
            } finally {
                //关闭response
                try {
                    response.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
                try {
                    httpClient.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
    
        }
    }

    六、HttpConfig

    package com.mwq.crawler.test;
    
    import org.apache.http.client.config.RequestConfig;
    import org.apache.http.client.methods.CloseableHttpResponse;
    import org.apache.http.client.methods.HttpGet;
    import org.apache.http.impl.client.CloseableHttpClient;
    import org.apache.http.impl.client.HttpClients;
    import org.apache.http.util.EntityUtils;
    
    import java.io.IOException;
    
    public class HttpConfigTest {
        public static void main(String[] args) {
            //创建httpjclient 对象
            CloseableHttpClient httpClient = HttpClients.createDefault();
    
            //创建HttpGet对象,设置URL访问地址
            HttpGet httpGet = new HttpGet("http://www.itcast.cn");
                //配置请求信息
            RequestConfig config= RequestConfig.custom().setConnectTimeout(1000) //创建连接的最长时间,单位毫秒
                  .setConnectionRequestTimeout(500) //设置获取连接的最长时间,单位是毫秒
                  .setSocketTimeout(10*1000) //设置数据传输的最长时间,单位是毫秒
                  .build();
            //给请求设置请求信息
            httpGet.setConfig(config);
            //使用HttpClient发起请求
            CloseableHttpResponse response = null;
            try {
                response = httpClient.execute(httpGet);
                if (response.getStatusLine().getStatusCode() == 200) {
                    String content = EntityUtils.toString(response.getEntity());
                    System.out.println(content.length());
                }
            } catch (IOException e) {
                e.printStackTrace();
            } finally {
                //关闭response
                try {
                    response.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
                try {
                    httpClient.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
    
        }
    }

    七、HttpClientPool

    package com.mwq.crawler.test;
    
    import org.apache.http.client.methods.CloseableHttpResponse;
    import org.apache.http.client.methods.HttpGet;
    import org.apache.http.impl.client.CloseableHttpClient;
    import org.apache.http.impl.client.HttpClients;
    import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
    import org.apache.http.util.EntityUtils;
    
    import java.io.IOException;
    
    public class HttpClientPoolTest {
        public static void main(String[] args) {
            //创建连接池管理器
            PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager();
    
            //设置连接最大连接数
            cm.setMaxTotal(100);
            //设置每个主机的最大连接数
            cm.setDefaultMaxPerRoute(10);
            //使用连接池发请求
            doGet(cm);
            doGet(cm);
        }
    
        private static void doGet(PoolingHttpClientConnectionManager cm) {
            //不是每次创建新的HttpClient,而是从连接池中获取HttpClient对象
            CloseableHttpClient httpClient = HttpClients.custom().setConnectionManager(cm).build();
            HttpGet httpGet = new HttpGet("http://www.itcast.cn");
            CloseableHttpResponse response = null;
            try {
                response = httpClient.execute(httpGet);
                if(response.getStatusLine().getStatusCode() == 200){
                    String content  = EntityUtils.toString(response.getEntity(),"utf8");
                    System.out.println(content.length());
    
                }
            } catch (IOException e) {
                e.printStackTrace();
            }finally {
                if(response !=null){
                    try {
                        response.close();
                    } catch (IOException e) {
                        e.printStackTrace();
                    }
                    //不能关闭,由连接池管理
    
                }
            }
        }
    
    }

    八、Jsoup

    package jsoup;
    /**
     * Jsoup解详文件,一般不用其爬取,因为其对一些技术支持不如httpclient号
     * 最终都是转为dom对象进行解析
     * 解析为元素后可以获取元素中的数据
     * selector查找元素单一方式,组合方式
     */
    
    import org.apache.commons.io.FileUtils;
    import org.jsoup.Jsoup;
    import org.jsoup.nodes.Attributes;
    import org.jsoup.nodes.Document;
    import org.jsoup.nodes.Element;
    import org.jsoup.select.Elements;
    import org.junit.Test;
    
    import java.io.File;
    import java.io.IOException;
    import java.net.MalformedURLException;
    import java.net.URL;
    import java.util.Set;
    
    public class JsoupFirstTest {
    
            @Test
            public void testUrl()throws Exception{
                //解析URL地址,第一个参数是访问的URL,第二个url是访问超时时间
                Document doc = Jsoup.parse(new URL("http://www.itcast.cn"), 1000);
                //使用 标签选择器,获取title标签中的内容
                String title = doc.getElementsByTag("title").first().text();
                //dayin
                System.out.println(title);
            }
            @Test
            public void testString() throws Exception{
                //使用工具类读取文件,获取字符串
                String content = FileUtils.readFileToString(new File(""),"utf8");
                //解析字符串
                Document doc = Jsoup.parse(content);
                String title = doc.getElementsByTag("title").first().text();
                System.out.println(title);
            }
            @Test
            public void testFile() throws Exception{
                //解析文件
                Document doc = Jsoup.parse(new File(""), "utf8");
                String title = doc.getElementsByTag("title").first().text();
                System.out.println(title);
            }
            @Test
            public  void testDom() throws Exception{
                //解析文件获取document 对象
                Document doc = Jsoup.parse(new File(""), "utf8");
                //根据ID查询元素
                Element element = doc.getElementById("city_di");
                //根据标签获取元素
                Element element1 = doc.getElementsByTag("span").first();
                //根据CLASS获取元素
                Elements el = doc.getElementsByClass("class_a class_b");
                //根据属性获取元素
                Elements abc = doc.getElementsByAttribute("abc");
                Element href = doc.getElementsByAttributeValue("href", "http://sh.itcase.cn").first();
                //打印元素内容
                System.out.println(element.text());
                System.out.println(element1);
                System.out.println(el);
                System.out.println(href);
            }
            @Test
            public void testData() throws Exception{
                //解析文件,获取doc对象
                Document doc = Jsoup.parse(new File(""), "utf8");
                //根据ID获取元素
                Element element=doc.getElementById("test");
                String str = "";
                //从元素中获取数据
                //从元素中获取id
                str = element.id();
                //从元素中获取classname
                str = element.className();
                Set<String> classSet = element.classNames();
                for(String s :classSet){
                    System.out.println(s);
                }
                //从元素中获取属性值
               str =  element.attr("id");
                //从元素中获取所有属性
                Attributes attributes = element.attributes();
                System.out.println(attributes.toString());
                //从元素中获取文本内容
                str = element.text();
                System.out.println(str);
    
    
            }
            @Test
        public void testSelecter() throws Exception{
                //解析HTML文件,获取dom对象
                Document doc = Jsoup.parse(new File(""), "utf8");
                //通过标签查找元素
    //            Elements span = doc.select("span");
    //            for (Element element : span) {
    //                System.out.println(element.text());
    //            }
    //            Element element = doc.select("#city_bj").first();
    //            System.out.println(element.text());
                //通过class查找元素
                Element e = doc.select(".class_a").first();
                //通过属性查找元素
                Element element = doc.select("[abc]").first();
                //利用属性值查找元素
                Elements elements = doc.select("[class-s_name]");
                for (Element element1 : elements) {
                    System.out.println(element1.text());
                }
    
            }
            @Test
        public void testSelector2()throws Exception{
                //解析HTML文件,获取dom对象
                Document doc = Jsoup.parse(new File(""), "utf8");
                //元素+id
                Element select = doc.select("h3#city_bj").first();
                System.out.println(select.text());
                //元素+class
                select = doc.select("li.class_a").first();
                //元素+属性名
                select = doc.select("span[abc]").first();
                //任意组合
                select = doc.select("span[abc].s_name").first();
                //ancestor child:查找某个元素下子元素,比如:city_con li
                Elements select1 = doc.select(".city_con li");
                for (Element element : select1) {
                    System.out.println(element.text());
                }
                //parent > child:查找某个父元素下的直接子元素,如:
                //。city_con > ul >li 查找city_con第一级(直接子元素)的ul,在找所有ul下的第一级Li,ul和Li都是标签
                select1 = doc.select(".city_con > ul >li");
                //parent > *:查找父元素下所有直接子元素
                select1 = doc.select(".city_con > ul >*");
            }
    
    }
  • 相关阅读:
    ASP.NET State Service
    C# winform 打印当前窗体
    js计算时间之差
    转 Javascript中event.keyCode键码值表收藏
    从客户端中检测到有潜在危险的 Request.Form
    从VS2005项目转换为VS2008项目 转载
    C# Windows DataGridView 判断CheckBox 选取的方法
    在Visual Studio 2005中调试SQL Server 2005的存储过程
    ASP.NET Ajax 中出现的 sys 未定义(sys undefined)解决方法总结
    KubeMeet 直播 | 现场直击大规模集群、混合环境下的云原生应用交付难题
  • 原文地址:https://www.cnblogs.com/mwq1992/p/14218971.html
Copyright © 2011-2022 走看看