方法一:
直接在spider文件下设置代理,通过传参的方式设置在Request中
import scrapy class MimvpSpider(scrapy.spiders.Spider): name = "mimvp" allowed_domains = ["mimvp.com"] start_urls = [ "http://proxy.mimvp.com/exist.php", "https://proxy.mimvp.com/exist.php", ] ## 代理设置方式1:直接在代理里设置 def start_requests(self): urls = [ "http://proxy.mimvp.com/exist.php", "https://proxy.mimvp.com/exist.php", ] for url in urls: meta_proxy = "" if url.startswith("http://"): meta_proxy = "http://180.96.27.12:88" # http代理 elif url.startswith("https://"): meta_proxy = "http://109.108.87.136:53281" # https代理 yield scrapy.Request(url=url, callback=self.parse, meta={'proxy': meta_proxy}) def parse(self, response): mimvp_url = response.url # 爬取时请求的url body = response.body # 返回网页内容 print("mimvp_url : " + str(mimvp_url)) print("body : " + str(body))
方法二:
利用中间件middlewares.py+settings配置
middlewares.py
## 代理设置方式2: 通过 middlewares.py + settings.py 配置文件设置 ## mimvp custom by yourself class ProxyMiddleware(object): def process_request(self,request,spider): if request.url.startswith("http://"): request.meta['proxy']="http://180.96.27.12:88" # http代理 elif request.url.startswith("https://"): request.meta['proxy']="http://109.108.87.136:53281" # https代理 # # proxy authentication # proxy_user_pass = "USERNAME:PASSWORD" # encoded_user_pass = base64.encodestring(proxy_user_pass) # request.headers['Proxy-Authorization'] = 'Basic ' + encoded_user_pass
settings.py
# Enable or disable downloader middlewares # See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html # 优先级越小越先执行 DOWNLOADER_MIDDLEWARES = { 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 110, 'mimvp_proxy_python_scrapy.middlewares.ProxyMiddleware': 100, }