爬虫_糗事百科（正则表达式）

import requests
import re
from lxml import etree

headers = 'Mozilla/5.0 (Windows NT 10.0; …) Gecko/20100101 Firefox/60.0'


def get_html(url):
    try:
        response = requests.get(url, headers)
        response.raise_for_status()
        response.encoding = 'utf-8'
        return response.text
    except:
        print('get_html() faild')


def parse_html(html):
    # 正则表达式
    hrefs = re.findall(r'<a class="contentHerf" href=(.*?)', html, re.S)
    # xpath
    element = etree.HTML(html)
    hrefs = element.xpath('//a[@class="contentHerf"]/@href')
    
    contents = re.findall(r'<div class="content".*?<span>(.*?)</span>', html, re.S)
    
    new_content = []
    for content in contents:
        content = re.sub('<.*?>', '', content)
        x = content.strip()
        new_content.append(x)
    return new_content


def main():
    page_num = 13
    for i in range(1, page_num+1):
        url = 'https://www.qiushibaike.com/text/page/{}/'.format(i)
        html = get_html(url)
        contents = parse_html(html)
        print(contents)


if __name__ == '__main__':
    main()

hrefs = re.findall(r'<a class="contentHerf" href=(.*?)', html, re.S)

这个链接的正则表达式感觉写的没问题啊，可是匹配不到东西，奇了怪了，先放着吧，xpath可以匹配到

运行结果