Java-爬虫(Jsoup、EasyExcel)

Java爬虫

一直没机会用过java写爬虫，把之前收藏的链接拿出来运行下，记录。

1. 使用Jsoup获取页面信息(和Python的BeautifulSoup类似)
2. 解析页面信息存储
3. EasyExcel转成信息存储

运行结果：

运行代码：

package org.jsoup.utils;

import java.io.IOException;

/**
 * @ProjectName: DoubanGet
 * @Package: org.jsoup.utils
 * @Description:
 * @Author: huyuqiao
 * @CreateDate: 2020/9/27 19:04
 */
public class DoubanGet {
    public static void main(String[] args) throws IOException {
        String url = "https://book.douban.com/latest?icn=index-latestbook-all";
        EasyExcelUtils.simpleWrite(BookInfoUtils.getBookInfoList(url));
    }

}

基本实体类：

package org.jsoup.utils;

/**
 * @ProjectName: BookDetailInfo
 * @Package: org.jsoup.utils
 * @Description:
 * @Author: huyuqiao
 * @CreateDate: 2020/9/27 18:58
 */
import lombok.Builder;
import lombok.Data;
import lombok.ToString;

@Data
@Builder
@ToString
public class BookDetailInfo {

    private String author;
    private String authorUrl;
    private String price;
}

package org.jsoup.utils;

/**
 * @ProjectName: BookEntity
 * @Package: org.jsoup.utils
 * @Description:
 * @Author: huyuqiao
 * @CreateDate: 2020/9/27 18:58
 */
import lombok.Builder;
import lombok.Data;
import lombok.ToString;

@Data
@Builder
@ToString
public class BookEntity {

    private String id;
    private String bookName;
    private String starsCount;
    private String author;
    private String bookImgUrl;
    //    private String price;
    private String detailPageUrl;
    private String description;
    private BookDetailInfo bookDetailInfo;
}

package org.jsoup.utils;

import com.alibaba.excel.annotation.ExcelProperty;
import lombok.Builder;
import lombok.Data;

/**
 * @ProjectName: ColumnData
 * @Package: org.jsoup.utils
 * @Description:
 * @Author: huyuqiao
 * @CreateDate: 2020/9/27 18:59
 */
@Data
@Builder
public class ColumnData {

    @ExcelProperty("书名称")
    private String bookName;

    @ExcelProperty("评分")
    private String starsCount;

    @ExcelProperty("作者")
    private String author;

    @ExcelProperty("封面图片")
    private String bookImgUrl;

    @ExcelProperty("简介")
    private String description;

    @ExcelProperty("单价")
    private String price;
}

获取页面信息:

package org.jsoup.utils;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

/**
 * @ProjectName: BookInfoUtils
 * @Package: org.jsoup.utils
 * @Description:
 * @Author: huyuqiao
 * @CreateDate: 2020/9/27 18:55
 */
public class BookInfoUtils {

    public static List<BookEntity> getBookInfoList(String url) throws IOException {
        List<BookEntity>  bookEntities=new ArrayList<>();
        Document doc = Jsoup.connect(url).get();
        Elements liDiv = doc.select("#content > div > div.article > ul > li");
        for (Element li : liDiv) {
            Elements urls = li.select("a[href]");
            Elements imgUrl = li.select("a > img");
            Elements bookName = li.select(" div > h2 > a");
            Elements starsCount = li.select(" div > p.rating > span.font-small.color-lightgray");
            Elements author = li.select("div > p.color-gray");
            Elements description = li.select(" div > p.detail");

            String bookDetailUrl = urls.get(0).attr("href");
            BookDetailInfo detailInfo = getDetailInfo(bookDetailUrl);
            BookEntity bookEntity = BookEntity.builder()
                    .detailPageUrl(bookDetailUrl)
                    .bookImgUrl(imgUrl.attr("src"))
                    .bookName(bookName.html())
                    .starsCount(starsCount.html())
                    .author(author.text())
                    .bookDetailInfo(detailInfo)
                    .description(description.html())
                    .build();
//            System.out.println(bookEntity);
            bookEntities.add(bookEntity);
        }
        return bookEntities;
    }
    /**
     *
     * @param detailUrl
     * @return
     * @throws IOException
     */
    public static BookDetailInfo getDetailInfo(String detailUrl)throws IOException {

        Document doc = Jsoup.connect(detailUrl).get();
        Elements content = doc.select("body");

        Elements price = content.select("#buyinfo-printed > ul.bs.current-version-list > li:nth-child(2) > div.cell.price-btn-wrapper > div.cell.impression_track_mod_buyinfo > div.cell.price-wrapper > a > span");
        Elements author = content.select("#info > span:nth-child(1) > a");
        BookDetailInfo bookDetailInfo = BookDetailInfo.builder()
                .author(author.html())
                .authorUrl(author.attr("href"))
                .price(price.html())
                .build();
        return bookDetailInfo;
    }
}

转成excel

package org.jsoup.utils;

import com.alibaba.excel.EasyExcel;

import java.util.ArrayList;
import java.util.List;

/**
 * @ProjectName: EasyExcelUtils
 * @Package: org.jsoup.utils
 * @Description:
 * @Author: huyuqiao
 * @CreateDate: 2020/9/27 18:59
 */
public class EasyExcelUtils {

    public static void simpleWrite(List<BookEntity> bookEntityList) {
        String fileName = "C:\Users\v-huyuqiao\Desktop\datalist" + System.currentTimeMillis() + ".xlsx";
        EasyExcel.write(fileName, ColumnData.class).sheet("书本详情").doWrite(data(bookEntityList));
        System.out.println("excel文件生成完毕...");
    }
    private static List<ColumnData> data(List<BookEntity> bookEntityList) {
        List<ColumnData> list = new ArrayList<>();
        bookEntityList.forEach(b -> {
            ColumnData data = ColumnData.builder()
                    .bookName(b.getBookName())
                    .starsCount(b.getStarsCount())
                    .author(b.getBookDetailInfo().getAuthor())
                    .bookImgUrl(b.getBookImgUrl())
                    .description(b.getDescription())
                    .price(b.getBookDetailInfo().getPrice())
                    .build();
            list.add(data);
        });
        return list;
    }
}