python 自建爬虫复用简单框架(gevent异步)

一般爬虫可以分为以下几个步骤：

一、打开指定网页

二、解析网页

三、处理/存储数据，新增任务网页

另外异步的话，需要调度器。

简单爬虫的话，不需要搞复杂验证码，requests/urllib修改cookie,header就能访问的话，写一个打开，一个解析就够了，处理数据和新任务，直接写在解析类就下，gevent也可以直接异步。

_{项目路径：ur'D:python_pymy_scrapy/scrapy_tools'}

# scrapy_tools下添加__init__.py作为包使用

itemparse.py

按照数据的结构建立相应的xpath 结构

# -*- coding: utf-8 -*-
"""
Created on Fri Jul 07 17:24:34 2017

@author: willowj
"""
import sys
stdout, stdin, stderr = sys.stdout, sys.stdin, sys.stderr
reload(sys)
sys.stdout, sys.stdin, sys.stderr = stdout, stdin, stderr
sys.setdefaultencoding('utf8')


import gevent
import pandas as pd 
import numpy as np
from lxml import html
import time
import codecs
import json


def list_0e(list_):
    if isinstance(list_, list):
        if not list_:
            return None
        else:
            if len(list_)>1:
                print 'warning : list>1,list[1]:', list_[1] #,len(list_)
            return list_[0]
    else:
        return list_


class ItemParse(object):
    """docstring for zhihu_topi"""
    name = 'ItemParse'

    base_url = 'https://www.zhihu.com/topic/19551147/top-answers'
    pageN_x = '//div[@class="zm-invite-pager"]//span[last()-1]/a/text()'
    new_urls_x = None

    #以下一条数据的节点，以及每一项
    items_node_x = '//div[@class="feed-main"]'
    #注意每一条数据的属性都是在一个项目里搜索，xpath '.'开头
    item_xs = dict(
        question_name = '''.//a[@class='question_link']/text()''', 
        #question_href = '''.//a[@class='question_link']/@href''', 
        author = './/div[@data-action="/answer/content"]/@data-author-name',
        author_href = '''.//a[@class='author-link']/@href''',  
        ups_x = './/div[@class="zm-item-vote-info"]/@data-votecount',
        answers_text = ".//textarea/text()",
        commentN = './/a[@name="addcomment"]/text()[last()]',
        entry_url = './/div[@data-action="/answer/content"]/@data-entry-url',

        #re:
        #z = re.compile('.')
        )    
    
    #换页url样式
    def getnextpages(self):
        if self.pageN > 1:
        #自定义换也规则，只有一页则为 False
            urls = [self.base_url + '?page=%s' %n 
                        for n in range(self.pageN,1,-1)
                    ]
            return urls


    def __init__(self, html_):
        #self.item_atrr_xpath()
        self.results = []
        self.new_urls = []
        self.pageN = self.update_page_n(html_)
        self.nextpages = self.getnextpages()
        self.parase(html_)


    def parase(self, html_):
        #优先使用xpath,,补充使用re; 找不到的item 返回none
        etree = html.document_fromstring(html_)
        items_nodes = etree.xpath(self.items_node_x)
        #results = []
        for ee in items_nodes:
            ee_str = None
            ite = {}
            for item,itemx in self.item_xs.items():
                # re, or xpath
                if hasattr(itemx, 'findall'):
                    if ee_str is None:
                        ee_str = html.to_string(ee)
                    ite[item] = itemx.findall(ee_str)
                #xpath   
                elif isinstance(itemx, str) or isinstance(itemx, unicode): 
                    if itemx.startswith('./'):
                        ite[item] = ee.xpath(itemx)
                    else:
                        print item
                        raise 'xpath not startwith ./'
                else:
                    print item
                    raise 'not re.pattarn object or xpath str'
                
                if len(ite[item]) == 0:
                    ite[item] = None
                elif len(ite[item]) == 1:
                    ite[item] = ite[item][0]
                else:
                    ite[item] = '
'.join([str(__i) for __i in ite[item]])
                
            self.results.append(ite)
        
        #new_url
        if self.new_urls_x:
            self.new_urls.extend(etree.xpath(self.new_urls_x)) 

    #获取有多少页
    def update_page_n(self, html_):
        if self.pageN_x:
            etree = html.document_fromstring(html_)
            pages = etree.xpath(self.pageN_x)
            pages = list_0e(pages)
            if isinstance(pages, str): 
                pages.strip()
            if pages and pages.isdigit():
                return int(pages)
        else:
            return 1

    #普通的获取项目下所有换页
    def get_nextpages(self, opener, sleep_sec=None):
        for url in self.nextpages:
            if sleep_sec:
                time.sleep(sleep_sec)
            #if not hasattr(opener, 'get')    
            _re = opener.get(url)
            print _re.status_code,  _re.url
            self.parase(_re.text)
            print time.time()
    #暂时把 异步控制和存储方法写到了这里
    #gevent 协程方法            
    def __gevent_get_nextpages(self, opener):
        print id(opener)
        while self.nextpages:
            #start_time = time.time()
            url = self.nextpages.pop()
            print gevent.getcurrent()
            zhihu_re = opener.get(url)
            #gevent.sleep(5)
            print zhihu_re.status_code,  url
            self.parase(zhihu_re.text) 
            print time.time()
    #gevent 协程方法
    def get_nextpages_by_gevent(self, opener_class, g_n=4):
        '''
        param:  opener_class : 创建网页打开器的类
                g_n: 协程数量，默认4个
        '''
        from gevent import monkey; monkey.patch_all()
          
        start_time = time.time()
        gs = [gevent.spawn(self.__gevent_get_nextpages, opener_class())
                for i in range(g_n)
                ]
        gevent.joinall(gs)    

        print time.time() - start_time 
        self.save_to_excel()

    def save_to_excel(self, path=None):  
        if path:  
            save_name = path
        else:     
            save_name = u''+ self.name 
                           + time.strftime('%Y%m%d_%H_%M', time.localtime()) 
                           + '.xlsx'
        print save_name
        result_pd = pd.DataFrame(self.results)
        print 'pd ok'
        result_pd.to_excel(u'' + save_name, encoding='gb18030')        
        print 'saved to ' + save_name


    def save_to_json(self, path=None):
        if path:  
            save_name = path
        else:     
            save_name = u''+ self.name 
                           + time.strftime('%Y%m%d_%H_%M', time.localtime()) 
                           + '.json'
        print save_name
        with codecs.open(save_name, 'w', encdoing='gb18030') as f:
            f.write(josn.dumps(self.results))
            
        print 'saved to '+ save_name

 使用时继承类重写类属性和getnextpages 换页方法

web_opener.py

使用requests.Session，保持会话的方式速度大概会快一倍

 对应gevent异步，多少个协程就会生成同等的会话，各自打开网页互补干扰。 方法暂时写在itemparse.py

# -*- coding: utf-8 -*-
"""
2017年8月17日星期四
下午 17:22
@author: willowj
"""
import sys
sys.setdefaultencoding('utf8')  

import requests
#from requests.cookies import (
#    cookiejar_from_dict, extract_cookies_to_jar, RequestsCookieJar, merge_cookies)

class SessionFopener(object):
    """requests 封装的网页打开器
    param: headers 默认使用类属性，实例化的时候自己可以传入
           cookie_dic 默认禁用
           proxies 默认无
    """
    headers = {
        'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
        'Accept-Encoding':'gzip, deflate, sdch',
        'Accept-Language':'zh-CN,zh;q=0.8,en;q=0.6',
        'Cache-Control':'max-age=0',
        'Connection':'keep-alive',
        #'Cookie':'q'
        #'Host':'www.zhihu.com',
        'Upgrade-Insecure-Requests':'1',
        'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36',
        }

    def __init__(self, headers=None, cookie_dic=None, proxies=None):
    
        self.req_s = requests.Session()
        self.req_s.adapters.DEFAULT_RETRIES = 3
        self.req_s.keep_alive = True 

        if headers:
            self.req_s.headers = headers 
        else:     
            self.req_s.headers = self.headers

        if not cookie_dic:
            cookie_dic = {}
        self.req_s.cookies = requests.cookies.cookiejar_from_dict(cookie_dic)

        if proxies:
            self.req_s.proxies = proxies

    def close(self):
        self.req_s.close()

    def get(self, *arg, **karg):
        return self.req_s.get(*arg, **karg)

    def post(self, *arg, **karg):
        return self.req_s.post(*arg, **karg)

    def set_cookiejar(self, cookie_dic={}):
        self.req_s.cookies = requests.cookies.cookiejar_from_dict(cookie_dic)

    def add_cookiejar(self, cookie_dic):
        self.req_s.cookies = requests.cookies.merge_cookies(self.req_s.cookies, cookie_dic)


    def set_headers(self, headers={}):
        self.req_s.headers = headers

    def add_headers(self, headers_dic):
        for k,v in header_dic:
            self.req_s.headers[k] = v


    def set_proxies(self, proxies):
        self.req_s.proxies = proxies


    @classmethod    
    def cookiejar_from_dict(cls, cookie_dic):
        return requests.cookies.cookiejar_from_dict(cookie_dic)

    def __enter__(self):
        print 'enter'
        return self

    def __exit__(self, *used):
        self.req_s.close()
        del self.req_s
        print 'exit'


if __name__ == '__main__':
    with  SessionFopener() as req_o:
        res_p = req_o.get('http://httpbin.org/get')
        
    print res_p.json()  

大众点评店铺爬取示例：

只需要继承后重写解析的节点、换页的url形式就行

暂时未考虑外链接。

# -*- coding: utf-8 -*-
"""
Created

2017年8月17日星期四
下午 19:33

@author: Administrator
"""
import sys
stdout, stdin, stderr = sys.stdout, sys.stdin, sys.stderr
reload(sys)
sys.stdout, sys.stdin, sys.stderr = stdout, stdin, stderr
sys.setdefaultencoding('utf8')  

sys.path.append(ur'D:python_pymy_scrapy')
from scrapy_tools.web_opener import SessionFopener
from scrapy_tools.itemparse import ItemParse

class DzdpItemParse(ItemParse):
    """广州酒家(文昌店)的点评
    docstring for zhihu_topi"""
    name = u'DzdpItemParse广州酒家'

    base_url = 'https://www.dianping.com/shop/516983/review_more'
    pageN_x = ".//a[@class='PageLink'][last()]/text()"
    new_urls_x = None

    #以下一条数据的节点，以及每一项
    items_node_x = './/div[@class="comment-list"]/ul/li'
    #注意每一条数据的属性都是在一个项目里搜索，xpath '.'开头
    item_xs = dict(
        user_id = '''.//*[@class="J_card"]/@user-id''',
        #question_href = '''.//a[@class='question_link']/@href''' ,

        comm_per = """.//span[@class='comm-per']/text()""",
        total_mark = """.//*[@class="user-info"]/span[1]/@class""",
        taste = """.//*[@class="comment-rst"]/span[1]/text()""",
        environment = """.//*[@class="comment-rst"]/span[2]/text()""",
        sevice = """.//*[@class="comment-rst"]/span[3]/text()""",

        comments_agree = '''.//span[@class="heart-num"]/text()''',
        comment_text = """.//*[@class="J_brief-cont"]/text()""",
        comment_date = '''.//*[@class="time"]/text()''',
        recommend_food = 
        u'''.//*[@class="comment-recommend" 
        and (contains(text(),推荐) 
        or contains(text(),喜欢))]
        [1]/a/text()'''
        #  中文得使用unicode                                                        
        #re:
        #z = re.compile('.')
        )    

    
    def getnextpages(self):
        if self.pageN > 1:
        #自定义换也规则，只有一页则为 False
            urls = [self.base_url + '?pageno=%s' %n
                    for n in range(self.pageN, 1, -1)
                    ]
            return urls


open_s = SessionFopener()   #实例化一个打开器
respon_= open_s.get(DzdpItemParse.base_url) #打开初始页
gzjj_item = DzdpItemParse(respon_.text) #解析对象用初始页html实例化

#同步方式的话，使用普通方法
gzjj_item.get_nextpages(open_s, sleep_sec=None)

#异步方法:
#gzjj_item.get_nextpages_by_gevent(SessionFopener) #实例异步方法

结果：本来打开一个网页0.5279 s，开四个协程后77.71s爬完613个页面，平均0.13s一个，速度提升至4倍

200 https://www.dianping.com/shop/516983/review_more?pageno=600
1503074965.07
<Greenlet at 0x9c44620: <bound method DzdpItemParse.__gevent_get_nextpages of <__main__.DzdpItemParse object at 0x094BF470>>(<scrapy_tools.web_opener.SessionFopener object at )>
200 https://www.dianping.com/shop/516983/review_more?pageno=602
1503074965.1
<Greenlet at 0x9c445d0: <bound method DzdpItemParse.__gevent_get_nextpages of <__main__.DzdpItemParse object at 0x094BF470>>(<scrapy_tools.web_opener.SessionFopener object at )>
200 https://www.dianping.com/shop/516983/review_more?pageno=601
1503074965.14
<Greenlet at 0x9c44670: <bound method DzdpItemParse.__gevent_get_nextpages of <__main__.DzdpItemParse object at 0x094BF470>>(<scrapy_tools.web_opener.SessionFopener object at )>
200 https://www.dianping.com/shop/516983/review_more?pageno=604
1503074965.54
<Greenlet at 0x9c44440: <bound method DzdpItemParse.__gevent_get_nextpages of <__main__.DzdpItemParse object at 0x094BF470>>(<scrapy_tools.web_opener.SessionFopener object at )>
200 https://www.dianping.com/shop/516983/review_more?pageno=607
1503074965.59
<Greenlet at 0x9c44670: <bound method DzdpItemParse.__gevent_get_nextpages of <__main__.DzdpItemParse object at 0x094BF470>>(<scrapy_tools.web_opener.SessionFopener object at )>
200 https://www.dianping.com/shop/516983/review_more?pageno=605
1503074965.64
<Greenlet at 0x9c44620: <bound method DzdpItemParse.__gevent_get_nextpages of <__main__.DzdpItemParse object at 0x094BF470>>(<scrapy_tools.web_opener.SessionFopener object at )>
200 https://www.dianping.com/shop/516983/review_more?pageno=606
1503074965.67
<Greenlet at 0x9c445d0: <bound method DzdpItemParse.__gevent_get_nextpages of <__main__.DzdpItemParse object at 0x094BF470>>(<scrapy_tools.web_opener.SessionFopener object at )>
200 https://www.dianping.com/shop/516983/review_more?pageno=611
1503074966.1
<Greenlet at 0x9c445d0: <bound method DzdpItemParse.__gevent_get_nextpages of <__main__.DzdpItemParse object at 0x094BF470>>(<scrapy_tools.web_opener.SessionFopener object at )>
200 https://www.dianping.com/shop/516983/review_more?pageno=609
1503074966.15
<Greenlet at 0x9c44670: <bound method DzdpItemParse.__gevent_get_nextpages of <__main__.DzdpItemParse object at 0x094BF470>>(<scrapy_tools.web_opener.SessionFopener object at )>
200 https://www.dianping.com/shop/516983/review_more?pageno=610
1503074966.18
<Greenlet at 0x9c44620: <bound method DzdpItemParse.__gevent_get_nextpages of <__main__.DzdpItemParse object at 0x094BF470>>(<scrapy_tools.web_opener.SessionFopener object at )>
200 https://www.dianping.com/shop/516983/review_more?pageno=608
1503074966.22
<Greenlet at 0x9c44440: <bound method DzdpItemParse.__gevent_get_nextpages of <__main__.DzdpItemParse object at 0x094BF470>>(<scrapy_tools.web_opener.SessionFopener object at )>
200 https://www.dianping.com/shop/516983/review_more?pageno=612
1503074966.7
200 https://www.dianping.com/shop/516983/review_more?pageno=614
1503074966.74
200 https://www.dianping.com/shop/516983/review_more?pageno=615
1503074967.05
200 https://www.dianping.com/shop/516983/review_more?pageno=613
1503074967.09
77.7100000381
DzdpItemParse广州酒家20170819_00_49.xlsx
pd ok
saved to DzdpItemParse广州酒家20170819_00_49.xlsx

分布式多进程、入数据库的话，还得单独写调度器、与数据对接的模块