手动爬虫之淘宝笔记本栏（ptyhon3）

1.这次爬虫用到了之前封装的Url_ProxyHelper类，源代码如下

import urllib.request as ur

class Url_ProxyHelper:
    def __init__(self, url, proxy_add, savepath=None):
        self.url = url
        self.proxy_add = proxy_add
        self.req = None
        self.proxy = None
        self.opener = None
        self.info = None
        self.save_path = savepath

    # 报头代理设置
    def set_UrlAndProxy(self):
        # 添加报头
        self.req = ur.Request(self.url)
        self.req.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:52.0) Gecko/20100101 Firefox/52.0')

        # 设置代理服务器
        self.proxy = ur.ProxyHandler({'http': self.proxy_add})
        self.opener = ur.build_opener(self.proxy, ur.HTTPHandler)
        ur.install_opener(self.opener)
        return self.req

    # 数据存档
    def save_InFile(self):
        self.req = self.set_UrlAndProxy()
        self.info = ur.urlopen(self.req).read()
        open(self.save_path, 'wb').write(self.info)

    # 数据返回
    def feedbak_info(self):
        self.req = self.set_UrlAndProxy()
        self.info = ur.urlopen(self.req).read().decode('utf-8')  # decode()用来解码，特别是中文
        return str(self.info)

2.爬取源代码：

import urllib.request as ur
import urllib.error as ue
from Url_ProxyHelper import Url_ProxyHelper
import re

# 设置目标网址 quote()函数能够解决url中出现的中文所导致的一些解析问题
url = ur.quote("https://s.taobao.com/list?q=平板电脑&q=平板电脑&s=",  safe='/:?=', encoding='utf-8')
# 设置存放路径
save_path = "E:/workspace/PyCharm/codeSpace/books/python_web_crawler_book/chapter6/demo2/images/"
# 设置代理服务器IP
proxy_add = "218.73.139.196:808"

def craw(url, save_path, proxy_add,page):
    url = url+str((page-1)*48)
    # 调用Url_ProxyHelper封装类
    uph = Url_ProxyHelper(url, proxy_add)
    infos = uph.feedbak_info()
    # 设置正则表达式 一般来讲先把这个实例拿出来，然后根据实例写通式
    pattern = '"(pic_url)":"(.+?.jpg)'
    infos = re.compile(pattern=pattern).findall(infos)
    x = 1
    for info in infos:
        image_name = save_path+str(page)+"_"+str(x)+".jpg"
        image_url = "http:"+info[1]
        try:
            ur.urlretrieve(image_url, filename=image_name)
        except ue.HTTPError as e:
            if hasattr(e, 'code'):
                print(e.code)
            if hasattr(e, 'reason'):
                print(e.reason)
        except ue.URLError as e:
            if hasattr(e, 'code'):
                print(e.code)
            if hasattr(e, 'reason'):
                print(e.reason)
        x += 1

# 只爬取了第一页
craw(url, save_path, proxy_add, 1)