20 古诗文网站诗文爬取（正则方法）

"""古诗文网爬虫"""


import re
import requests

def parse_page(url):
    headers = {
        'User-Agent': 'Mozilla/5.0',
    }

    response = requests.get(url, headers)
    # print(response.text)
    text = response.text

    # re解析
    titles = re.findall(r'<divsclass="cont">.*?<b>(.*?)</b>', text, re.DOTALL)     # .本不会匹配
，加上参数re.DOTALL即对任何字符都有效
    # print(titles)
    dynasties = re.findall(r'<p class="source">.*?<a.*?>(.*?)</a>', text, re.DOTALL)
    # print(dynasties)
    authors = re.findall(r'<p class="source">.*?<a.*?>.*?<a.*?>(.*?)</a>', text, re.DOTALL)
    # print(authors)
    content_tags = re.findall(r'<div class="contson" .*?>(.*?)</div>', text, re.DOTALL)
    # print(content_tags)
    contents = []
    for content in content_tags:
        x = re.sub(r'<.*>', "", content).strip()
        contents.append(x)
    poems = []
    for value in zip(titles, dynasties, authors, contents):
        title, dynasty, author, content = value
        poem = {
            'title': title,
            'dynasty': dynasty,
            'author': author,
            'content': content
        }
        poems.append(poem)
    
    # 输出诗文记录
    for poem in poems:
        print(poem)


def main():
    url = "https://www.gushiwen.org/default_{}.aspx"
    for x in range(1, 11):
        newurl = url.format(x)
        parse_page(newurl)

if __name__ == '__main__':
    main()