生产者和消费者模型

一.锁机制

1.普通锁

import threading,random,time
gMoney=1000
gTotalTimes=10
gTtimes=0
gLock=threading.Lock()

class Producer(threading.Thread):
    def run(self):
        global gMoney
        global gTtimes
        while True:
            money=random.randint(100,1000)
            gLock.acquire()
            if gTtimes>=gTotalTimes:
                gLock.release()
                break
            gMoney+=money
            print("%s挣了%d元钱，剩余%d元钱"%(threading.currentThread(),money,gMoney))
            gTtimes+=1
            gLock.release()
            time.sleep(0.5)

class Consumer(threading.Thread):
    def run(self):
        global gMoney
        while True:
            money=random.randint(100,1000)
            gLock.acquire()
            if gMoney>=money:
                gMoney-=money
                print("%s消费了%d元钱，剩余%d元钱"%(threading.current_thread(),money,gMoney))
            else:
                if gTtimes>=gTotalTimes:
                    gLock.release()
                    break
                print("%s准备消费%d元钱，但是余额不足"%(threading.current_thread(),money))
            gLock.release()
            time.sleep(0.5)

def main():
    for x in range(3):
        t=Consumer(name="消费者线程%d"%x)
        t.start()

    for x in range(5):
        t=Producer(name="生产者线程%d"%x)
        t.start()

if __name__=="__main__":
    main()

2.Condition

import threading
import random
from time import sleep


ct = threading.Condition()

all_money = 1000  # 基础金钱1000元
count = 10  # 限制生产者只可以生产十次


class producers(threading.Thread):
    '''生产者模式'''
    def run(self):
        global all_money
        global count

        while True:
            ct.acquire()  # 处理数据前，先上锁
            if count > 0:  # 如果生产次数小于十次
                money = random.randint(200,1000)  # 随机生产200-1000元
                all_money += money  # 总金钱数 = 原总金钱数+生产金钱数
                count -= 1  # 允许生产次数-1
                print('生产者%s生产了%d元，剩余金钱%d元' % (threading.current_thread(), money, all_money))
            else:  # 如果生产次数已满10次
                ct.release()  # 解锁
                break  # 生产结束，跳出循环
            ct.notify_all()  # 通知所有等待中的消费者，生产已完成，可以开始消费
            ct.release()  # 解锁
            sleep(0.5)


class comsumer(threading.Thread):
    '''消费者模式'''
    def run(self):
        global all_money
        global count

        while True:
            ct.acquire()  # 处理数据前，先上锁
            money = random.randint(200,1000)  # 随机消费200-1000元
            # 下面这个while是重点！（敲黑板，记笔记，后面我会说到的）
            while money > all_money:  # 如果需消费金额大于总金额，则等待至总金额大于需消费金钱
                if count == 0:  # 如果生产者生产次数已达上限
                    ct.release()  # 结束前解锁
                    return  # 结束函数
                print('消费者%s需要消费%d元，剩余金钱%d元，不足' % (threading.current_thread(), money, all_money))
                ct.wait()  # 进入等待（阻塞进程）
            all_money -= money  # 剩余金额大于消费金额时，总金额 = 原总金额 - 消费金额
            print('消费者%s消费了%d元，剩余金钱%d元' % (threading.current_thread(), money, all_money))
            ct.release()  # 解锁
            sleep(0.5)


if __name__ == '__main__':
    for i in range(3):
        th = comsumer(name='线程%d'%i)
        th.start()

    for i in range(5):
        th = producers(name='线程%d'%i)
        th.start()

 3.Queue

import threading
from queue import Queue
import time


def set_value(qu):
    '''生成元素放入列队'''
    index = 0
    while True:
        qu.put(index)
        index += 1
        start = time.time()
        time.sleep(2)  # 每隔三秒放入一个
        end = time.time()
        print('阻塞时间为：',end-start,'秒threading.Thread')


def get_value(qu):
    while True:
        print('数据：',qu.get())  # 列队中有数据就取出来，没有就等待


if __name__ == '__main__':
    qu = Queue(4)
    t1 = threading.Thread(target=set_value,args=[qu])
    t2 = threading.Thread(target=get_value,args=[qu])

    t1.start()
    t2.start()

二.多线程下载表情包

1.不用多线程

import requests,re
from lxml import etree
from urllib import request
def get_page(url):
    header={
        'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36'
    }
    response=requests.get(url,headers=header)
    text=response.text
    html=etree.HTML(text)
    imgs=html.xpath('//div[@class="page-content text-center"]//a[@class="col-xs-6 col-sm-3"]//img[@class!="gif"]')
    for img in imgs:
        img_url=img.get("data-original") #获取图片URL
        suffix=img_url.split(".")[-1]   #获取后缀名JPG或者GIF
        alt=img.get("alt")              #获取名字
        alt=re.sub(r'[?.*,!！？。]',"",alt)  #替换名字里的掉特殊字符
        filename=alt+"."+suffix     
        request.urlretrieve(img_url,"imgs/"+filename)

def main():
    for x in range(10):
        url="http://www.doutula.com/photo/list/?page=%s" %(x)
        get_page(url)
        
if __name__=="__main__":
    main()

 2.利用多线程

 main()

• 定义两个队列，和创建多线程
• page_queue()：存放每一页的url
• img_queue()：存放每一页里面所有的表情的url

Producer()

• 从page_queue()队列中去每一页的url，直到队列为空则break
• 用xpath提取出每一页的所有图片的url
• 把每个图片的url和名字存放到img_queue()队列里面

Consumer()

• 从img_queue()队列中取出图片的url和名字
• 下载保存
• 直到page_queue()和img_queue()两个队列都为空则break

import requests
from lxml import etree
from urllib import request
import os
import re
import threading
from queue import Queue

class Producer(threading.Thread):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36',
        'Referer': 'https://movie.douban.com/'
    }

    def __init__(self, page_queue, img_queue, *args, **kwargs):
        super(Producer, self).__init__(*args, **kwargs)
        self.page_queue = page_queue
        self.img_queue = img_queue

    def run(self):
        while True:
            if self.page_queue.empty():
                break
            url = self.page_queue.get()
            self.parse_page(url)

    def parse_page(self,url):
        response = requests.get(url,headers=self.headers)
        text = response.text
        html = etree.HTML(text)
        imgs = html.xpath("//div[@class='page-content text-center']//img[@class!='gif']")
        for img in imgs:
            # print(etree.tostring(img))
            #图片地址
            img_url = img.get('data-original')
            #图片名字
            alt = img.get('alt')
            #替换掉名字里面的特殊字符
            alt = re.sub(r'[?？.，。！!*]','',alt)
            #获取图片的后缀名（.gif .jpg）
            suffix = os.path.splitext(img_url)[1]
            #保存的时候完整的图片名字
            filename = alt + suffix
            self.img_queue.put((img_url,filename))


class Consumer(threading.Thread):
    def __init__(self,page_queue,img_queue,*args,**kwargs):
        super(Consumer, self).__init__(*args,**kwargs)
        self.page_queue = page_queue
        self.img_queue = img_queue

    def run(self):
        while True:
            if self.img_queue.empty() and self.page_queue.empty():
                break
            img_url,filename = self.img_queue.get()
            request.urlretrieve(img_url, 'C:/Users/Administrator/Desktop/images/' + filename)
            print("已下载完一张图片")


def main():
    page_queue = Queue(1000)
    img_queue = Queue(10000)

    for x in range(1,1758):
        url = 'http://www.doutula.com/photo/list/?page=%d'%x
        page_queue.put(url)

    for x in range(10):
        t = Producer(page_queue,img_queue)
        t.start()

    for x in range(10):
        t = Consumer(page_queue,img_queue)
        t.start()

if __name__ == '__main__':
    main()