手动爬虫之京东笔记本栏（ptyhon3）

import urllib.request as ur
import urllib.error as ue
import re
# 目标网址
url = 'https://list.jd.com/list.html?cat=670,671,672'
# 存放路径
save_path = 'E:/workspace/PyCharm/codeSpace/books/python_web_crawler_book/chapter6/demo1/images/'
# 代理服务器ip
proxy_add = '115.174.66.148:8118'

def get_JD_pictures(url, save_path, proxy_add, page):
    # 根据页面设置url
    url = url+"&page="+str(page)
    # 添加报头
    req = ur.Request(url)
    req.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:52.0) Gecko/20100101 Firefox/52.0')

    # 设置代理
    proxy = ur.ProxyHandler({'http': proxy_add})
    opener = ur.build_opener(proxy, ur.HTTPHandler)
    ur.install_opener(opener)

    # 爬取页面
    info = ur.urlopen(req).read()
    # 信息存档
    info = str(info)
    pattern_1 = '<div id="plist".+? <div class="page clearfix">'
    info = re.compile(pattern=pattern_1).findall(info)
    info = info[0]
    pattern_2 = '<img width="220" height="220" data-img="1" src="//(.+?.jpg)">'
    image_list = re.compile(pattern=pattern_2).findall(info)
    x = 1
    for image_url in image_list:
        image_name = save_path+str(page)+"_"+str(x)+".jpg"
        image_url = "http://"+image_url
        try:
            ur.urlretrieve(image_url, filename=image_name)
        except ue.HTTPError as e:
            if hasattr(e, 'code'):
                print(e.code)
            if hasattr(e, 'reason'):
                print(e.reason)
        except ue.URLError as e:
            if hasattr(e, 'code'):
                print(e.code)
            if hasattr(e, 'reason'):
                print(e.reason)
        x += 1

get_JD_pictures(url, save_path, proxy_add, 1)