目前学习的爬取小数据图片zzz

import os
import threading
import re
import time
from lxml import etree


all_img_urls = []    # 图片列表页面的数组

g_lock = threading.Lock()      # 初始化一个锁

# 声明一个生产者的类，来不断地获取图片详情页地址，然后添加到 all_img_url列表中

# url = "http://www.xiaohuar.com/"

all_urls = []

class Spider(object):
    # 构造函数，初始化数据实用
    def __init__(self,target_url,headers):
        self.target_url = target_url
        self.headers = headers

    # 获取所有的想要抓取的URL
    def getUrls(self,start_page,page_num):
        for i in range(start_page,page_num):
            url = self.target_url % i
            all_urls.append(url)


if __name__ == '__main__':
    headers = {
                "User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.62 Safari/537.36",
                "Host":"eclick.baidu.com",

            }
    target_url = "http://www.xiaohuar.com/list-1-%d.html"   # 抓取链接的样式

    spider = Spider(target_url,headers)    # 抓取链接的对象传入 链接与请求头
    spider.getUrls(0,14)    # 抓取的多少页面的链接
    # print (all_urls)


class Producer(threading.Thread):    #创建一个生产者用来批量的'生产'链接

    def run(self):
        headers = {
            "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.62 Safari/537.36",
            "Host": "eclick.baidu.com",

        }

        while len(all_urls) > 0:    # 这里写了一个死循环，为的是能够一直抓取为爬去数据的链接
            g_lock.acquire()     # 锁，为的是不让不同的线程共同使用同一条连接
        # for url in all_urls:
            url = all_urls.pop()    # 使用pop方法，可以获取链接
            g_lock.release()      # 获取连接后 释放锁，让其他线程可前去列表中获取链接
            response = requests.get(url,headers).text

            selector = etree.HTML(response)    # 使用xpath

            mods = selector.xpath("//div[@class='item_t']")    # 获取指定标签

            for i in mods:
                img_link = i.xpath("div[@class='img']/a/img/@src")
                name = i.xpath("div[@class='img']/span/text()")
                name = name[0].encode("utf8")
                img_link = img_link[0].encode("utf8")

                comment = {name: img_link}
                if img_link.startswith("/"):    # 因为抓取的链接，有一部分是本地，所以在此处将之拼接成可直接访问的url
                    str = "http://www.xiaohuar.com"
                    img_link = str + img_link
                    comment = {name: img_link}
                    all_img_urls.append(comment)
                all_img_urls.append(comment)


for x in range(10):     # 创建10个线程用来爬去链接
    down = Producer()
    down.run()
# print all_img_urls




class DownPic(threading.Thread):      # 用来下载爬取数据的类

    def run(self):
        headers = {
            "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.62 Safari/537.36",
            "Host": "eclick.baidu.com",

        }

        while True:  # 这个地方写成死循环，为的是不断监控图片链接数组是否更新
            g_lock.acquire()
            if len(all_img_urls) == 0: #没有图片了，就解锁
                g_lock.release()
                continue
            else:
                img = all_img_urls.pop()
                g_lock.release()
                # 遍历字典列表
                for key,value in img.items():
                    path =  "xiaohua/%s.jpg"% key.decode("utf8")
                    response = requests.get(value)
                    # print path
                    with open (path,"wb") as f:
                        f.write(response.content)
                        f.close()#
# #
#
#
for x in range(10):
    down = DownPic()
    down.run()