爬虫_古诗文网(队列，多线程，锁，正则，xpath)

import requests
from queue import Queue
import threading
from lxml import etree
import re
import csv


class Producer(threading.Thread):
    headers = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.75 Safari/537.36'}
    def __init__(self, page_queue, poem_queue, *args, **kwargs):
        super(Producer, self).__init__(*args, **kwargs)
        self.page_queue = page_queue
        self.poem_queue = poem_queue


    def run(self):
        while True:
            if self.page_queue.empty():
                break
            url = self.page_queue.get()
            self.parse_html(url)


    def parse_html(self, url):
        # poems = []
        headers = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.75 Safari/537.36'}
        response = requests.get(url, headers=headers)
        response.raise_for_status()
        html = response.text
        html_element = etree.HTML(html)
        titles = html_element.xpath('//div[@class="cont"]//b/text()')
        contents = html_element.xpath('//div[@class="contson"]')
        hrefs = html_element.xpath('//div[@class="cont"]/p[1]/a/@href')
        for index, content in enumerate(contents):
            title = titles[index]
            content = etree.tostring(content, encoding='utf-8').decode('utf-8')
            content = re.sub(r'<.*?>|
|', '', content)
            content = re.sub(r'u3000u3000', '', content)
            content = content.strip()
            href = hrefs[index]
            self.poem_queue.put((title, content, href))


class Consumer(threading.Thread):

    def __init__(self, poem_queue, writer, gLock, *args, **kwargs):
        super(Consumer, self).__init__(*args, **kwargs)
        self.writer = writer
        self.poem_queue = poem_queue
        self.lock = gLock

    def run(self):
        while True:
            try:
                title, content, href = self.poem_queue.get(timeout=20)
                self.lock.acquire()
                self.writer.writerow((title, content, href))
                self.lock.release()
            except:
                break


def main():
    page_queue = Queue(100)
    poem_queue = Queue(500)
    gLock = threading.Lock()
    fp = open('poem.csv', 'a',newline='', encoding='utf-8')
    writer = csv.writer(fp)
    writer.writerow(('title', 'content', 'href'))
   

    for x in range(1, 100):
        url = 'https://www.gushiwen.org/shiwen/default.aspx?page=%d&type=0&id=0' % x
        page_queue.put(url)

    for x in range(5):
        t = Producer(page_queue, poem_queue)
        t.start()

    for x in range(5):
        t = Consumer(poem_queue, writer, gLock)
        t.start()

if __name__ == '__main__':
    main()

运行结果