scrapy

1、基本命令：

Global commands: scrapy + (大的命令):
    scrapy startproject ***  #创建项目  ***(这是项目名，不是爬虫名) 
    cd myproject     
    scrapy genspider baidu www.baidu.com  
        # 创建爬虫程序baidu.py，以及爬虫名称
        # 百度网址是域名，不一定非要有，有的话只能爬百度开头的网址
    　　# 一个项目可以有多个爬虫，使用 scrapy list 查看
    scrapy settings    
        #如果是在项目目录下，则得到的是该项目的配置    
    scrapy runspider baidu.py  
        # 爬虫文件的绝对路径（不是项目文件） 
    　　# 运行一个独立的python文件，不必创建项目 scrapy shell http://www.baidu.com
        # 在交互式调试，如选择器规则正确与否 response.status / text / body 
        # response.xpth()
    scrapy  view(response)         
        # 下载到本地，完毕后直接弹出浏览器，以此可以分辨出哪些数据是ajax请求

Project-only commands:
    scrapy crawl  baidu --nolog (爬虫名称)  
        #运行爬虫，必须创建项目才行，确保配置文件中ROBOTSTXT_OBEY = False
    scrapy check        
        #检测项目中有无语法错误
    scrapy list         
        #列出项目中所包含的爬虫名
    scrapy parse http://www.baodu.com/ --callback parse    
        #以此可以验证我们的回调函数是否正确
    scrapy bench        
        #scrapy bentch压力测试,看每分钟爬多少页面
    官网链接
        https://docs.scrapy.org/en/latest/topics/commands.html    

2、custom_settings

custom_settings值为一个字典，定义一些配置信息，在运行爬虫程序时，这些配置会覆盖项目级别的配置。所以custom_settings必须被定义成一个类属性(放在parse之前)，由于settings会在类实例化前加载，但是后来居上custom_settings会覆盖settings.py的配置

import scrapy
from scrapy.selector import HtmlXPathSelector
from scrapy.http.request import Request
from ..items import * 
from scrapy.http.cookies import CookieJar

class zhihuSpiser(scrapy.Spider):
    name="zhihu"
    allowed_urls=["xxx",]
    start_urls=["xxxx",]
    custom_settings={ "DEFAULT_REQUEST_HEADERS":{
                “user-agent”：...
                "accept_language":...

                    }     
                    }
    cookie_dict={}

3、start_requests 两种写法 

def start_requests(self):
    for url in self.start_urls:
        yield Request(url=url,callback=self.parse)

def start_requests(self):
    req_list = []
    for url in self.start_urls:
        req_list.append(Request(url=url,callback=self.parse2))
    return req_list  # scrapy内部会将返回值(list可迭代对象)转换成迭代器。

4、cookie处理

4.1、post请求

from scrapy.http import Request 

req = Request(
    url='http://dig.chouti.com/login',
    method='POST',
    body='phone=8613121758648&password=woshiniba&oneMonth=1',
    headers={'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8'},
    cookies={},
        # meta={'cookiejar': True} 或者 cookies=self.cookie_dict
    callback=self.parse_check_login,
            )

4.2、手动操作cookie

def parse(self,response):
     
    prin(response.headers.getlist('Set-Cookie'))    # 原始cookie
    
    from scrapy.http.cookies import CookieJar    # 解析之后的cookie
    cookie_jar = CookieJar()
        cookie_jar.extract_cookies(response, response.request)
        for k, v in cookie_jar._cookies.items():
            for i, j in v.items():
                for m, n in j.items():
                    self.cookie_dict[m] = n.value
        print(cookie_dict)    
    req = Request(
            url='http://dig.chouti.com/login',
            method='POST',
            headers={'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8'},
            body='phone=8615131255089&password=pppppppp&oneMonth=1',
            cookies=self.cookie_dict,
            callback=self.check_login
        )
    yield req
    
def check_login(self, response):
     yield Request(
                url='http://dig.chouti.com/link/vote?linksId=%s' %(link_id,),
                method='POST',
                cookies=self.cookie_dict,
                callback=self.do_favor
                    )   
                    
def do_favor(self, response):
    print(response.text)

4.3、自动操作cookie ：  meta={'cookiejar':True}

配置文件制定是否允许操作cookie:
    # Disable cookies (enabled by default)
    # COOKIES_ENABLED = False   # 第36行

class ChoutiSpider(scrapy.Spider):
    name = 'chouti'
    allowed_domains = ['chouti.com']
    start_urls = ['http://dig.chouti.com/',]

    def start_requests(self):
        for url in self.start_urls:
            yield Request(url=url,callback=self.parse_index,meta={'cookiejar':True})
            
    def parse_index(self,response):
        req = Request(
            url='http://dig.chouti.com/login',
            method='POST',
            headers={'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8'},
            body='phone=8613121758648&password=woshiniba&oneMonth=1',
            callback=self.parse_check_login,
            meta={'cookiejar': True}
        )
        yield req

    def parse_check_login(self,response):
        # print(response.text)
        yield Request(
            url='https://dig.chouti.com/link/vote?linksId=19440976',
            method='POST',
            callback=self.parse_show_result,
            meta={'cookiejar': True},
        )

    def parse_show_result(self,response):
        print(response.text)

4.4、cookie其他用法

模拟浏览器登录

start_requests()方法，可以返回一个请求给爬虫的起始网站，这个返回的请求相当于start_urls，start_requests()返回的请求会替代start_urls里的请求

Request()get请求，可以设置，url、cookie、回调函数

FormRequest.from_response()表单post提交，第一个必须参数，上一次响应cookie的response对象，其他参数，cookie、url、表单内容等

yield Request()可以将一个新的请求返回给爬虫执行


在发送请求时cookie的操作，
meta={'cookiejar':1}表示开启cookie记录，首次请求时写在Request()里
meta={'cookiejar':response.meta['cookiejar']}表示使用上一次response的cookie，写在FormRequest.from_response()里post授权
meta={'cookiejar':True}表示使用授权后的cookie访问需要登录查看的页面

 

获取Scrapy框架Cookies

请求Cookie
Cookie = response.request.headers.getlist('Cookie')
print(Cookie)

响应Cookie
Cookie2 = response.headers.getlist('Set-Cookie')
print(Cookie2)

# -*- coding: utf-8 -*-
import scrapy
from scrapy.http import Request,FormRequest

class PachSpider(scrapy.Spider):                            #定义爬虫类，必须继承scrapy.Spider
    name = 'pach'                                           #设置爬虫名称
    allowed_domains = ['edu.iqianyue.com']                  #爬取域名
    # start_urls = ['http://edu.iqianyue.com/index_user_login.html']     #爬取网址,只适于不需要登录的请求，因为没法设置cookie等信息

    header = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:54.0) Gecko/20100101 Firefox/54.0'}  #设置浏览器用户代理

    def start_requests(self):       #用start_requests()方法,代替start_urls
        """第一次请求一下登录页面，设置开启cookie使其得到cookie，设置回调函数"""
        return [Request('http://edu.iqianyue.com/index_user_login.html',meta={'cookiejar':1},callback=self.parse)]

    def parse(self, response):     #parse回调函数

        data = {                    #设置用户登录信息，对应抓包得到字段
            'number':'adc8868',
            'passwd':'279819',
            'submit':''
            }

        # 响应Cookie
        Cookie1 = response.headers.getlist('Set-Cookie')   #查看一下响应Cookie，也就是第一次访问注册页面时后台写入浏览器的Cookie
        print(Cookie1)

        print('登录中')
        """第二次用表单post请求，携带Cookie、浏览器代理、用户登录信息，进行登录给Cookie授权"""
        return [FormRequest.from_response(response,
                                          url='http://edu.iqianyue.com/index_user_login',   #真实post地址
                                          meta={'cookiejar':response.meta['cookiejar']},
                                          headers=self.header,
                                          formdata=data,
                                          callback=self.next,
                                          )]
    def next(self,response):
        a = response.body.decode("utf-8")   #登录后可以查看一下登录响应信息
        # print(a)
        """登录后请求需要登录才能查看的页面，如个人中心，携带授权后的Cookie请求"""
        yield Request('http://edu.iqianyue.com/index_user_index.html',meta={'cookiejar':True},callback=self.next2)
    def next2(self,response):
        # 请求Cookie
        Cookie2 = response.request.headers.getlist('Cookie')
        print(Cookie2)

        body = response.body  # 获取网页内容字节类型
        unicode_body = response.body_as_unicode()  # 获取网站内容字符串类型

        a = response.xpath('/html/head/title/text()').extract()  #得到个人中心页面
        print(a)

5、xpath

    # from scrapy.selector import HtmlXPathSelector
    # hxs=HtmlXPathSelector(response=response)
    # div_tag=hxs.xpath("//div[@id='content-list']/div[@class='item']")
    
    # 可以直接response.xpath()    没必要调用HtmlXPathSelector
    response.xpath('//div[@id='content-list']/div[@class='item']')
    
    for div in div_tag:
        a_text=div.xpath(".//div[@class='part1']/a[1]/text()").extract_first()
        a_href=div.xpath(".//div[@class='part1']/a[1]/@href").extract_first(default="not found")
        print(a_text,a_href)
        
        from ..items import * 
        item=XianglongItem(title=a_text,href=a_href)
        yield item
        
        pages=hxs.xpath().extract()
        for page in pages:
            page_url="xxxxx"+page_url
            yield Request(url=page_url,callback=self.parse)
        # 这样子会不停的爬取，可以在settings中设置DEPTH_LIMIT = 1

6、pipeline　　处理item或者DropItem

ITEM_PIPELINES={ TextPipeline: 300,
                 MongoPipeline:400,
                }
MONGO_URL="localhost"
MONGO_DB="quotes"

from scrapy.exceptions import DropItem

class TextPipeline(object):
    def __init__(self):
        self.limit=50
    def process_item(self,item,spider):
        if item["text]:
            if len(item["text]) > self.limit:
                item["text"] = item["text"][0:self.limit]/rstrip() + "..."
                return item
        else：
            from scrapy.exceptions import DropItem
            raise DropItem("Missing Text")

from scrapy.exceptions import DropItem

class MongoPipeline(objects):
    def __init__(self,mongo_url,mongo_db):
        self.mongo_url=mongo_url
        self.mongo_db=mongo_db

    @classmethod
    def from_crawler(cls,crawler):     
        # 一般用来获取settings信息，也可用于spider.py中
    
        return cls(
                mongo_url=crawler.settings.get(“MONGO_UR”),
                mongo_db=crawler.settings.get(“MONGO_DB”),        
             )   
             # 获取的mongo_url又传到了def __init__ 中的参数
             # def __init__ 中再接收参数

    def open_spider(self,spider):

        self.client=pymongo.MongoClient(self.mongo_url)
        self.db=self.client[self.mongo_db]

    def process_item(self,item,spider):
        name=item.__class__.__name__
        self.db["quotes"].insert(dict(item))
        return item

    def close(self,spider):
        self.client.close()

import json
from scrapy.exceptions import DropItem

class JsonPipLine(object):
    def open_spider(self,spider):
        self.file=open("items.jl","wb")
        
    def process_item(self,item,spider):
        line=json.dumps(dict(item))+"
"        
        self.file.write(line)
        return item

    def close(self,spider):
        selc.file.close()

from scrapy.exceptions import DropItem

class DropPipLine(object):
    def __init__(self):
        self.ids_seen = set()
    def process_item(self,item,spider):
        if item["id"] in self.self.ids_seen:
            raise DropItem("missing")
        else: self.ids_seen.add(item["id"])
            return item

7、dupefilter

7.1、源码解析

　　scrapy默认使用 scrapy.dupefilter.RFPDupeFilter 进行去重

　　dupefilter.py（源码）---注意返回True或者false ,True表示已经访问过；False表示未访问过

DUPEFILTER_CLASS = 'scrapy.dupefilter.RFPDupeFilter'
DUPEFILTER_DEBUG = False
JOBDIR = "保存范文记录的日志路径，如：/root/"  # 最终路径为 /root/requests.seen

　　RFPDupeFilter继承BaseDupeFilter，再利用request_fingerprint

class RFPDupeFilter(BaseDupeFilter):
    def __init__(self,path=None,debug=False):
        self.fingerprints=set()      # 集合set
    
    def request_seen(self,request):   # 关键函数
        fp=self.request_fingerprint(request)  # request的指纹
        if fp in self.fingerprints:
            return True
            
        added=self.server.sadd(self.key,fp)   # k，v 插入,返回值added
        return added == 0          #如果added=0，说明已经存在这个请求，就不会再加入队列    
    
    def request_fingerprint(self,request):   # 获取指纹
        return request_fingerprint(request)

　　拓展：利用request_fingerprint，为请求创建唯一标识

    http://www.oldboyedu.com?id=1&age=2
    http://www.oldboyedu.com?age=2&id=1

    from scrapy.utils.request import request_fingerprint
    from scrapy.http import Request

    u1 = Request(url='http://www.oldboyedu.com?id=1&age=2')
    u2 = Request(url='http://www.oldboyedu.com?age=2&id=1')

    result1 = request_fingerprint(u1)
    result2 = request_fingerprint(u2)
    print(result1,result2)

7.2、自己写去重

class MyDupeFilter(BaseDupeFilter):
    def __init__(self):
        self.visited_url = set()

    @classmethod
    def from_settings(cls, settings):
        """
        初始化时，调用
        :param settings: 
        :return: 
        """
        return cls()

    def request_seen(self, request):
        """
        检测当前请求是否已经被访问过
        :param request: 
        :return: True表示已经访问过；False表示未访问过
        """
        if request.url in self.visited_url:
            return True
        self.visited_url.add(request.url)
        return False

    def open(self):
        """
        开始爬去请求时，调用
        :return: 
        """
        print('open replication')

    def close(self, reason):
        """
        结束爬虫爬取时，调用
        :param reason: 
        :return: 
        """
        print('close replication')

    def log(self, request, spider):
        """
        记录日志
        :param request: 
        :param spider: 
        :return: 
        """
        print('repeat', request.url)

DUPEFILTER_CLASS = 'xxxxxxx.dupe.MyDupeFilter'

from scrapy.dupefilter import BaseDupeFilter
from scrapy.utils.request import request_fingerprint
class MyDupeFilter(BaseDupeFilter):

def __init__(self):
    self.record = set()

@classmethod
def from_settings(cls, settings):
    return cls()

def request_seen(self, request):
    print('url开始去重了，request.url')
    ident=request_fingerprint(request)
    if ident in self.record:
        print('已经访问过了', request.url,ident)
        return True     
    self.record.add(ident)
    return False
        # 为什么返回 True/False , 因为scheduler.py
def open(self):  # can return deferred
    pass

def close(self, reason):  # can return a deferred
    pass

8、scheduler

# 将request对象全部放到内部维护的队列：self.q=deque()
# 将request对象全部放到硬盘维护的队列：文件操作 mq

from scrapy.core.scheduler import Scheduler

def enqueue_request(self, request):
    if not request.dont_filter and self.df.request_seen(request):
        self.df.log(request, self.spider)
        return False
    dqok = self._dqpush(request)
    if dqok:pass
    else:pass
    return True
    
# dont_filter参数解析：        
#(request.dont_filter=False)
    # self.df.request_seen(request):
    #  - True,已经访问  那么if 为真，return False
    #  - False，未访问  那么if 为假，dqok = self._dqpush(request)
# request.dont_filter=True，
    # 那么if 为假，dqok = self._dqpush(request)全部加入到调度器                    

def next_request(self):
    pass

9、下载中间件　　--- 请求头、代理、证书、cookie、

9.1、对爬虫中所有请求发送时，携带请求头？

方案一：在每个Request对象中添加一个请求头

配置： 
    DOWNLOADER_MIDDLEWARES = {
       'xianglong.middlewares.UserAgentDownloaderMiddleware': 543,
    }
编写类

class UserAgentDownloaderMiddleware(object):

@classmethod
def from_crawler(cls, crawler):                
    s = cls()
    return s

def process_request(self, request, spider):            
    request.headers['User-Agent'] = "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36"

    # return None # 继续执行后续的中间件的process_request
    # from scrapy.http import Request
    # return Request(url='www.baidu.com') # 重新放入调度器中，当前请求不再继续处理
    # from scrapy.http import HtmlResponse # 执行从最后一个开始执行所有的process_response
    # return HtmlResponse(url='www.baidu.com',body=b'asdfuowjelrjaspdoifualskdjf;lajsdf')

def process_response(self, request, response, spider):
    return response

def process_exception(self, request, exception, spider):
    pass

配置文件：
USER_AGENT = 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'

9.2、scrapy中如何添加代理？

scrapy.contrib.downloadermiddleware.httpproxy.HttpProxyMiddleware
import os
import scrapy
from scrapy.http import Request

class ChoutiSpider(scrapy.Spider):
    name = 'chouti'
    allowed_domains = ['chouti.com']
    start_urls = ['https://dig.chouti.com/']

    def start_requests(self):
        os.03['HTTP_PROXY'] = "http://192.168.11.11"

        for url in self.start_urls:
            yield Request(url=url,callback=self.parse)

    def parse(self, response):
        print(response)

import random
import base64
import six
def to_bytes(text, encoding=None, errors='strict'):
    """Return the binary representation of `text`. If `text`
    is already a bytes object, return it as-is."""
    if isinstance(text, bytes):
        return text
    if not isinstance(text, six.string_types):
        raise TypeError('to_bytes must receive a unicode, str or bytes '
                        'object, got %s' % type(text).__name__)
    if encoding is None:
        encoding = 'utf-8'
    return text.encode(encoding, errors)

class MyProxyDownloaderMiddleware(object):
    def process_request(self, request, spider):
        proxy_list = [
            {'ip_port': '111.11.228.75:80', 'user_pass': 'xxx:123'},
            {'ip_port': '120.198.243.22:80', 'user_pass': ''},
            {'ip_port': '111.8.60.9:8123', 'user_pass': ''},
            {'ip_port': '101.71.27.120:80', 'user_pass': ''},
            {'ip_port': '122.96.59.104:80', 'user_pass': ''},
            {'ip_port': '122.224.249.122:8088', 'user_pass': ''},
        ]
        proxy = random.choice(proxy_list)
        if proxy['user_pass'] is not None:
            request.meta['proxy'] = to_bytes("http://%s" % proxy['ip_port'])
            encoded_user_pass = base64.encodestring(to_bytes(proxy['user_pass']))
            request.headers['Proxy-Authorization'] = to_bytes('Basic ' + encoded_user_pass)
        else:
            request.meta['proxy'] = to_bytes("http://%s" % proxy['ip_port'])
配置：
    DOWNLOADER_MIDDLEWARES = {
       # 'xiaohan.middlewares.MyProxyDownloaderMiddleware': 543,
                                }
    

9.3、证书

Https访问时有两种情况：
1. 要爬取网站使用的可信任证书(默认支持)
    DOWNLOADER_HTTPCLIENTFACTORY = "scrapy.core.downloader.webclient.ScrapyHTTPClientFactory"
    DOWNLOADER_CLIENTCONTEXTFACTORY = "scrapy.core.downloader.contextfactory.ScrapyClientContextFactory"
    
2. 要爬取网站使用的自定义证书
# settings.py
    DOWNLOADER_HTTPCLIENTFACTORY = "scrapy.core.downloader.webclient.ScrapyHTTPClientFactory"
    DOWNLOADER_CLIENTCONTEXTFACTORY = "项目名.https.MySSLFactory"
    
# https.py
    from scrapy.core.downloader.contextfactory import ScrapyClientContextFactory
    from twisted.internet.ssl import (optionsForClientTLS, CertificateOptions, PrivateCertificate)
    
    class MySSLFactory(ScrapyClientContextFactory):
        def getCertificateOptions(self):
            from OpenSSL import crypto
            v1 = crypto.load_privatekey(crypto.FILETYPE_PEM, open('/Users/wupeiqi/client.key.unsecure', mode='r').read())
            v2 = crypto.load_certificate(crypto.FILETYPE_PEM, open('/Users/wupeiqi/client.pem', mode='r').read())
            return CertificateOptions(
                privateKey=v1,  # pKey对象
                certificate=v2,  # X509对象
                verify=False,
                method=getattr(self, 'method', getattr(self, '_ssl_method', None))
            )
            
            
其他：
    相关类
        scrapy.core.downloader.handlers.http.HttpDownloadHandler
        scrapy.core.downloader.webclient.ScrapyHTTPClientFactory
        scrapy.core.downloader.contextfactory.ScrapyClientContextFactory

10、爬虫中间件

class XiaohanSpiderMiddleware(object):
    # Not all methods need to be defined. If a method is not defined,
    # scrapy acts as if the spider middleware does not modify the
    # passed objects.
    def __init__(self):
        pass
    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        return s

    # 每次下载完成之后，未执行parse函数之前。
    def process_spider_input(self, response, spider):
        # Called for each response that goes through the spider
        # middleware and into the spider.

        # Should return None or raise an exception.
        print('process_spider_input',response)
        return None

    def process_spider_output(self, response, result, spider):
        # Called with the results returned from the Spider, after
        # it has processed the response.

        # Must return an iterable of Request, dict or Item objects.
        print('process_spider_output',response)
        for i in result:+
            yield i

    def process_spider_exception(self, response, exception, spider):
        # Called when a spider or process_spider_input() method
        # (from other spider middleware) raises an exception.

        # Should return either None or an iterable of Response, dict
        # or Item objects.
        pass

    # 爬虫启动时，第一次执行start_requests时，触发。（只执行一次）
    def process_start_requests(self, start_requests, spider):
        # Called with the start requests of the spider, and works
        # similarly to the process_spider_output() method, except
        # that it doesn’t have a response associated.

        # Must return only requests (not items).

        print('process_start_requests')
        for r in start_requests:
            yield r

11、扩展、信号

11.1、单纯扩展

extends.py 
    class MyExtension(object):
        def __init__(self):
            pass

        @classmethod
        def from_crawler(cls, crawler):
            obj = cls()
            return obj
配置：
    EXTENSIONS = {
        'xiaohan.extends.MyExtension':500,
    }

11.2、扩展+信号

from scrapy import signals

class MyExtension(object):
    def __init__(self):
        pass

    @classmethod
    def from_crawler(cls, crawler):
        obj = cls()
# 在爬虫打开时，触发spider_opened信号相关的所有函数：xxxxxxxxxxx
        crawler.signals.connect(obj.xxxxxxxxxxx1, signal=signals.spider_opened)
# 在爬虫关闭时，触发spider_closed信号相关的所有函数：xxxxxxxxxxx
        crawler.signals.connect(obj.uuuuuuuuuu, signal=signals.spider_closed)
        return obj

    def xxxxxxxxxxx1(self, spider):
        print('open')

    def uuuuuuuuuu(self, spider):
        print('close')

            return obj
            
配置：  #  62 行
EXTENSIONS = {
'xiaohan.extends.MyExtension':500,
            }

12、自定制命令

commands 文件夹 ，在其中创建 crawlall.py (此处文件名就是自定义的命令)在settings.py中添加配置： COMMANDS_MODULE = "sp3.commands"在项目目录执行命令：scrapy crawlall ( cmd 中直接执行)

from scrapy.commands import ScrapyCommand
from scrapy.utils.project import get_project_settings

class Command(ScrapyCommand):
    requires_project = True

    def syntax(self):
        return '[options]'

    def short_desc(self):
        return 'Runs all of the spiders'

    def run(self, args, opts):
        spider_list = self.crawler_process.spiders.list()
        
        for name in spider_list:
            self.crawler_process.crawl(name, **opts.__dict__)
        self.crawler_process.start()

def run(self, args, opts):
    from scrapy.crawler import CrawlerProcess
    CrawlerProcess.crawl
    CrawlerProcess.start
    """
        self.crawler_process对象中含有：_active = {d,}
    """
    self.crawler_process.crawl('chouti',**opts.__dict__)
    self.crawler_process.crawl('cnblogs',**opts.__dict__)
    #
    self.crawler_process.start()