urllib2.HTTPError: HTTP Error 400: Bad Request

import urllib2
import re
import os
 
def process_item(self, item, spider):
        headers = {
                    "Host":    'img31.mtime.cn',
                    "User-Agent":    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:53.0) Gecko/20100101 Firefox/53.0',
                    "Accept":    'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
                    "Accept-Language":   'zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3',
                    "Accept-Encoding":    'gzip, deflate',
                    "Connection":    'keep-alive',
                    "Upgrade-Insecure-Requests": "1",          
                }
        
        req = urllib2.Request(url=item['addr'], headers=headers) 
        res = urllib2.urlopen(req)

        saveFilePath = os.path.join(os.path.curdir, "down_pic", item['name'].split("_")[0]+"_"+str(item["picCount"]))
        if os.path.exists(saveFilePath):
            pass
        else:
            #os.mkdir(saveFilePath) #只能建单层文件夹
            os.makedirs(saveFilePath)
        file_name = os.path.join(saveFilePath, item['name'] + '.jpg')
        with open(file_name, 'wb') as fp:
            fp.write(res.read())
        with open("./savePath.txt", "a") as fh:
            fh.write(file_name+"
")

 用urllib2抓取图片的时候，部分request报HTTP Error 400: Bad Request

参照：https://stackoverflow.com/questions/8840303/urllib2-http-error-400-bad-request?answertab=active#tab-top

because you aren't escaping the string for a URL.

改用requests成功解决问题。

import os
import requests 

 def process_item(self, item, spider):
        headers = {
                    "Host":    'img31.mtime.cn',
                    "User-Agent":    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:53.0) Gecko/20100101 Firefox/53.0',
                    "Accept":    'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
                    "Accept-Language":   'zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3',
                    "Accept-Encoding":    'gzip, deflate',
                    "Connection":    'keep-alive',
                    "Upgrade-Insecure-Requests": "1",          
                }
        
        res = requests.get(item['addr'], headers=headers)
        #print res
        #print res.url
        #print res.headers
        #print res.content 二进制数据
        
        saveFilePath = os.path.join(os.path.curdir, "down_pic", item['name'].split("_")[0]+"_"+str(item["picCount"]))
        if os.path.exists(saveFilePath):
            pass
        else:
            #os.mkdir(saveFilePath) #只能建单层文件夹
            os.makedirs(saveFilePath)
        file_name = os.path.join(saveFilePath, item['name'] + '.jpg')
        with open(file_name, 'wb') as fp:
            fp.write(res.content)
        with open("./savePath.txt", "a") as fh:
            fh.write(file_name+"
")