python3爬虫 -----爬取百思不得姐信息-------http://www.budejie.com/

# -*- coding:utf-8 -*-
# author:zxy
# Date:2018-10-21

import request
from lxml import etree
import threading
from queue import Queue
import csv
import requests

class Produce(threading.Thread):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) '
                      'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36',
        'Cookie': '__cfduid=ddb28ef1934faef742f7fb8911d7b33bd1540080067; UM_distinctid=16693ece9945b2-0b031da4b19f32-333b5602-1fa400-16693ece9958e4;'
                  ' _ga=GA1.2.1950184368.1540080070; _gid=GA1.2.1249143498.1540080070; _gat=1'
    }
    def __init__(self,page_queue,joke_queue,*args,**kwargs):
        super(Produce, self).__init__(*args,**kwargs)
        self.base_domain="http://www.budejie.com"
        self.page_queue = page_queue
        self.joke_queue = joke_queue
    def run(self):
        while True:
            if self.page_queue.empty():
                break
            url=self.page_queue.get()
            self.parse_url(url)

    def parse_url(self,url):
        reponse=requests.get(url,headers=self.headers)
        text=reponse.text
        html=etree.HTML(text)
        descs=html.xpath("//div[@class='j-r-list-c-desc']")
        for desc in descs:
            jokes=desc.xpath(".//text()")
            joke="
".join(jokes).strip()
            link=self.base_domain+desc.xpath(".//a/@href")[0]
            self.joke_queue.put((joke,link))
        print("="*30+"第%s页下载完成！"%url.split('/')[-1]+"="*30)


class Consumer(threading.Thread):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) '
                      'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36',
        'Cookie': '__cfduid=ddb28ef1934faef742f7fb8911d7b33bd1540080067; UM_distinctid=16693ece9945b2-0b031da4b19f32-333b5602-1fa400-16693ece9958e4;'
                  ' _ga=GA1.2.1950184368.1540080070; _gid=GA1.2.1249143498.1540080070; _gat=1'
    }
    def __init__(self,joke_queue,write,gLock,*args,**kwargs):
        super(Consumer, self).__init__(*args,**kwargs)
        self.joke_queue=joke_queue
        self.write=write
        self.gLock=gLock

    def run(self):
        while True:
            try:
                joke_info=self.joke_queue.get(timeout=40)
                joke,link=joke_info
                self.gLock.acquire()
                self.write.writerow((joke,link))
                self.gLock.release()
            except:
                break


def main():
    page_queue=Queue(100)
    joke_queue=Queue(1000)
    gLock=threading.Lock()
    fp=open('baisibudejie.csv','a',newline='',encoding='utf-8')
    writer=csv.writer(fp)
    writer.writerow(('content','link'))

    for x in range(1,11):
        url="http://www.budejie.com/%d"%x
        page_queue.put(url)

    for x  in range(5):
        t=Produce(page_queue,joke_queue)
        t.start()

    for x in range(3):
        t=Consumer(joke_queue,writer,gLock)
        t.start()


if __name__ == '__main__':
    main()