【Python学习之旅】---多线程爬取段子

import requests
import time
import threading
import queue
from lxml import etree
# "https://ishuo.cn/duanzi"
# header={"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.87 Safari/537.36"}
#爬取网页线程---爬取段子列表所在的网页，放入队列
class Mythread1(threading.Thread):
    def __init__(self,threaName,pageQueue,dataQueue):
        threading.Thread.__init__(self)
        self.threaName=threaName  #线程名
        self.pageQueue = pageQueue   #页码队列
        self.dataQueue = dataQueue   #数据队列
        self.headers={"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.87 Safari/537.36"}

    def run(self):
        print("启动线程",self.threaName)
        while not vlog1:
            try:
                page=self.pageQueue.get()
                url="https://ishuo.cn/duanzi"
                data=requests.get(url,headers=self.headers).text
                time.sleep(0.5)
                self.dataQueue.put(data)   #将数据放入到数据队列中
            except Exception as e:
                pass
        print("结束线程",self.threaName)


#解析网页线程---从队列中拿出列表网页，进行解析，并存储到本地
class Mythread2(threading.Thread):
    def __init__(self,threaName,dataQueue,filename):
        threading.Thread.__init__(self)
        self.threaName=threaName
        self.dataQueue = dataQueue
        self.filename = filename

    def run(self):
        print("启动线程",self.threaName)
        while not vlog1:
            try:
                data1=self.dataQueue.get()  #取出数据队列中的数据
                html=etree.HTML(data1)
                node_list=html.xpath('//div[@class="info"]/a') #获取这一页网页中所有符合条件的a标签
                # print(node_list)
                for node in node_list:
                    data2=node.text  #依次获取a标签的信息
                    self.filename.write(data2+"
") #将信息写入文件
            except Exception as e:
                pass
        print("结束线程",self.threaName)




vlog1=False  #判断页码队列中是否为空
vlog2=False  #判断数据队列中是否为空


def main():
    #页码队列
    pageQueue=queue.Queue(1)
    pageQueue.put(1)
    #存放采集结果的数据队列
    dataQueue=queue.Queue()
    #保存到文件
    filename=open(r"D:软件pythonpython_workPython_day18123.txt","a")
    #启动线程
    t1=Mythread1("采集线程",pageQueue,dataQueue)
    t1.start()
    t2=Mythread2("解析线程",dataQueue,filename)
    t2.start()
    #结束主线程
    #当pageQueue为空时，结束采集线程
    while not pageQueue.empty():
        pass
    global vlog1
    vlog1=True

    # 当dataQueue为空时，结束解析线程
    while not pageQueue.empty():
        pass
    global vlog2  #定义全局变量
    vlog2 =True

    t1.join()
    t2.join()
    filename.close()  #当2个线程执行完之后关闭文件
    print("结束！")



if __name__=='__main__':
    main()