一、线程回顾
import time import threading '''一、一个主线程''' # def sing(): # for i in range(1,6): # print('come baby 跟我一起 嗨 嗨 嗨 ！！！') # time.sleep(1) # # def dance(): # for i in range(1,6): # print('恰恰 肚皮 钢管舞 哈哈哈哈哈 ...... ') # time.sleep(1) # def main(): # sing() # dance() # if __name__ == '__main__': # main() '''二、面向过程创建线程：一个主线程，两个子线程''' # def sing(a): # for i in range(1,6): # print('当前线程：%s ...come %s 跟我一起 嗨 嗨 嗨 ！！！' %(threading.current_thread().name,a)) # time.sleep(1) # def dance(a): # for i in range(1,6): # print('当前线程：%s ... 恰恰 肚皮 钢管舞 %s你要哪一种 ' %(threading.current_thread().name,a)) # time.sleep(1) # def main(): # print('...联欢晚会现在开始...') # #创建唱歌线程 # a = '悟空' # t_sing = threading.Thread(target=sing,name='唱歌',args=(a,)) # # # 创建跳舞线程 # t_dance = threading.Thread(target=dance, name='跳舞',args=(a,)) # # #启动线程 # t_sing.start() # t_dance.start() # # #让主线程等待子线程执行完毕 # t_sing.join() # t_dance.join() # # print('晚会结束，各回各家') # if __name__ == '__main__': # main() '''三、面向对象创建线程''' #写一个类，继承threading.Thread class SingThread(threading.Thread): def __init__(self,name,a): super().__init__() self.name = name self.a = a def run(self): print("线程名：%s 参数：%s" %(self.name,self.a)) for i in range(1, 6): print('爱江山更爱美人...') time.sleep(1) class DanceThread(threading.Thread): def __init__(self, name, a): super().__init__() self.name = name self.a = a def run(self): print("线程名：%s 参数：%s" % (self.name, self.a)) for i in range(1, 6): print('蹦擦擦，蹦擦擦...') time.sleep(1) def main(): #创建线程 t_sing = SingThread('唱','八戒') t_dance = DanceThread('跳','悟能') #启动线程 t_sing.start() t_dance.start() #让主线程等待子线程执行完毕 t_sing.join() t_dance.join() if __name__ == '__main__': main()
二、队列
from queue import Queue #创建队列 q = Queue(5) #5个位子 print(q.empty()) #判断是否为空 #存入数据 q.put('浓眉哥') q.put('勒布朗') q.put('丹尼*格林') q.put('库兹马') q.put('麦基') print(q.full()) #判断是否满 print(q.qsize()) #返回队列大小 # q.put('波普',False) #如果队列满了，直接报错 # q.put('波普',True,3) #如果队列满了，等待3秒还没有空位，报错 #获取数据：先进先出 print(q.get()) print(q.get()) print(q.get()) print(q.get()) print(q.get()) # q.get('波普',False) #如果队列为空，直接报错 # q.get('波普',True,3) #如果队列为空，等待3秒还是空，报错
三、多线程爬虫
import time import threading from queue import Queue import requests from lxml import etree import json #存放采集线程 crawl_thread_list = [] #存放解析线程 parse_thread_list = [] def create_queue(): #创建页码队列 page_queue = Queue() for page in range(1,6): page_queue.put(page) # 创建内容队列 data_queue = Queue() return page_queue,data_queue class CrawlThread(threading.Thread): def __init__(self,name,page_queue,data_queue): super(CrawlThread,self).__init__() self.name = name self.page_queue = page_queue self.data_queue = data_queue self.url = 'http://www.fanjian.net/jiantu-{}' self.headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) ' 'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36'} def run(self): print('%s启动......' %self.name) while 1: if self.page_queue.empty(): break #从队列取出页码 page = self.page_queue.get() #拼接url url = self.url.format(page) #发送请求,拿到响应 r = requests.get(url=url,headers=self.headers) #将响应内容放到data_queue self.data_queue.put(r.text) break print('%s结束......' % self.name) class ParseThread(threading.Thread): def __init__(self,name,data_queue,fp,lock): super(ParseThread, self).__init__() self.name = name self.data_queue = data_queue self.fp = fp self.lock = lock def parse_content(self,data): tree = etree.HTML(data) '''先查找所有的li,再从li下查找图片标题和src''' li_list = tree.xpath('//ul[@class="cont-list"]/li') items = [] for l in li_list: # 获取图片标题 img_title = l.xpath('//h2/a/text()')[0] #获取图片url img_url = tree.xpath('//div[@class="cont-list-main"]/p/img/@data-src')[0] item = {'标题',img_title, '链接',img_url} items.append(item) #写入文件 self.lock.acquire() #上锁 for item in items: self.fp.write(str(item)) self.lock.release() #解锁 def run(self): while 1: print('%s启动......' % self.name) #从data_queue中取出一页数据 data = self.data_queue.get() #解析内容 self.parse_content(data) def create_crawl_thread(page_queue,data_queue): crawl_name = ['采集1号','采集2号','采集3号'] for name in crawl_name: #创建子线程 t_crawl = CrawlThread(name,page_queue,data_queue) #保存到列表 crawl_thread_list.append(t_crawl) def create_parse_thread(data_queue,fp,lock): parse_name = ['解析1号', '解析2号', '解析3号'] for name in parse_name: # 创建子线程 t_parse = ParseThread(name,data_queue,fp,lock) # 保存到列表 parse_thread_list.append(t_parse) def main(): # 创建队列 page_queue,data_queue = create_queue() #打开一个文件 fp = open('jiantu.txt','a',encoding='utf8') #创建锁 lock = threading.Lock() #创建采集线程 create_crawl_thread(page_queue,data_queue) #创建解析线程 create_parse_thread(data_queue,fp,lock) # 启动采集线程 for t in crawl_thread_list: t.start() # 启动解析线程 for t in parse_thread_list: t.start() # 让主线程等待子线程执行完毕 for t in crawl_thread_list: t.join() for t in parse_thread_list: t.join() #关闭文件 fp.close() print('主线程执行完毕！') if __name__ == '__main__': main()