案例_(单线程)使用正则的内涵段子爬虫

(单线程)使用正则的内涵段子爬虫

代码如下

from urllib.request import *
import re
import time


class Spider(object):
    def __init__(self):
        self.__start_page = int(input("请输入要爬取的开始页面:"))
        self.__end_page = int(input("请输入要爬取的结束页面:"))

        # 模拟浏览器代理
        self.__header = {
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3514.0 Safari/537.36"}

        # 用来记录帖子数量
        self.num = 1

    def __load_page(self, start_page, end_page):
        """
            下载页面
        """
        print("正在爬取中....")
        for page in range(self.__start_page, self.__end_page + 1):
            # 由于第一页是url比较特殊,要单独爬取
            if page == 1:
                url = "https://www.neihanba.com/dz/index.html"

            else:
                # 待爬取的地址
                url = "https://www.neihanba.com/dz/list_" + str(page) + ".html"

            # 发起请求
            request = Request(url, headers=self.__header)
            response = urlopen(request)

            # 获取每页的html源码字符串
            html = response.read().decode("gbk")

            # 获取所有符合条件的,返回一个列表
            content_list = re.findall(r'<div class="f18 mb20">.*?</div>', html)
            42             # 调用__deal_info()开始处理多余的信息
            self.__deal_info(content_list, page)

    def __deal_info(self, content_list, page):
        """
            处理每条的段子多余的部分
        """
        content = "                                 =============================第%d页=========================
" % page  # 用来拼接内容

        for info in content_list:
            # 注意 | 左右的空格不能随便加
            info = re.sub('(<div class="f18 mb20">)|(</div>)', "", info)

            content = content + "  %d、" % self.num + info + "
"
            self.num += 1

        # 调用write__page()开始写入数据
        self.__write_page(content)

    def __write_page(self, content):
        """
            把每个段子逐个写入文件里
        """
        with open("内涵段子.txt", "a") as f:
            f.write(content)

    def run(self):
        """
            控制爬虫运行
        """

        start_time = time.time()

        # 开始爬取
        self.__load_page(self.__start_page, self.__end_page)
        print("爬取完成...")
        end_time = time.time()
        print('用时:%.2f秒.' % (end_time - start_time))


if __name__ == '__main__':
    spider = Spider()
    spider.run()

结果预览如下:

如果你和我有共同爱好,我们可以加个好友一起交流哈!