【scrapy】scrapy-redis 全国建筑市场基本信息采集

简介

---

环境: python3.6

　　  scrapy 1.5

使用scrapy-redis 开发的分布式采集demo。一次简单的例子,供初学者参考(觉得有更好的方式麻烦反馈！)

源码地址:https://github.com/H3dg3h09/scrapy-redis-jzsc

目录

---

常规目录,存储用的mysql，文件一起传上去了。

static.py存放了mysql连接的类。其中写了（网上借鉴）一个根据item来入库的方法..非常方便了

from jzsc.settings import DB
import six

class Db():

    def __init__(self, host, port,user, password, db):
        import pymysql
        self._cnx = pymysql.connect(host=host, port=port, user=user, password=password, db=db, charset="utf8")
        self._cursor = self._cnx.cursor()

    def insert_data(self, table_name, data):
        '''
        :param table_name: str
        :param data: dict
        :return: bool
        '''

        col_str = ''
        row_str = ''
        for key in data.keys():
            col_str = col_str + " " + key + ","
            row_str = "{}'{}',".format(row_str,
                                       data[key] if "'" not in data[key] else data[key].replace("'", "\'"))
            sql = "INSERT INTO {} ({}) VALUES ({}) ON DUPLICATE KEY UPDATE ".format(table_name, col_str[1:-1],
                                                                                    row_str[:-1])
        for (key, value) in six.iteritems(data):
            sql += "{} = '{}', ".format(key, value if "'" not in value else value.replace("'", "\'"))
        sql = sql[:-2]

        self._cursor.execute(sql)  # 执行SQL
        i = self._cnx.insert_id()
        try:
            self._cnx.commit()  # 写入操作
        except AttributeError as e:
            raise e
        else:
            return i


    @classmethod
    def init_db(cls):
        host = DB.get('DATABASE_HOST', '')
        user = DB.get('DATABASE_USER', '')
        password = DB.get('DATABASE_PASSWORD', '')
        db = DB.get('DATABASE_DB', '')
        port = DB.get('DATABASE_PORT', '')
        return cls(host=host, port=port, user=user,    
                       password=password, db=db)
            

run.py脚本用于调试...我习惯写一个run在pycharm里打断点调试用

Spider

---

getid用于爬取列表页的url， 然后将url存入redis供baseinfo调用

*对于分页的处理可能比较笨..希望有朋友告诉我更优雅的方法,代码如下:

# -*- coding: utf-8 -*-
from scrapy_redis.spiders import RedisSpider
import redis
from jzsc.settings import REDIS_URL # redis配置
import logging
from scrapy import FormRequest
import time


class GetidSpider(RedisSpider):
    name = 'getid'
    redis_key = 'getidspider:start_urls'
    _rds = redis.from_url(REDIS_URL, db=0, decode_responses=True)
    custom_settings = {
        'LOG_FILE': 'jzsclog{name}_{t}.txt'.format(name=name, t=time.strftime('%Y-%m-%d', time.localtime()))
    }
    form_data = {
        '$reload': '0',
        '$pgsz': '15',
    }
    page = 1
    cookie = {}
    max_page = 0
    def parse(self, response):
        hrefs = response.xpath('//tbody[@class="cursorDefault"]/tr/td/a/@href').extract()

        for href in hrefs:
            new_url = response.urljoin(href)
            self._rds.rpush('baseinfospider:start_urls', new_url)

            logging.log(logging.INFO, "{url}".format(url=new_url))


            if not self.max_page: # 最大页数获取
                import re
                self.max_page = int(re.search(r'pc:d(.*?),', response.xpath('//a[@sf="pagebar"]').extract_first()).group(1))
                self.form_data['$total'] = re.search(r'tt:d(.*?),', response.xpath('//a[@sf="pagebar"]').extract_first()).group(1)
                self.cookies = eval(response.headers.get('cookies', {}))


            self.page += 1
            if self.page <= self.max_page:
                self.form_data['$pg'] = str(self.page)

                yield FormRequest(response.url, callback=self.parse,
                                  cookies=self.cookies,
                                  formdata=self.form_data, dont_filter=True)

baseinfo脚本用于抽取公司基本信息:

    def parse(self, response):
        base_div = response.xpath('//div[@class="user_info spmtop"]')[0]
        name = base_div.xpath('b/text()').extract_first()
        info = base_div.xpath('following-sibling::table[1]/tbody/tr/td/text()').extract()

        base_item = BasicInfoItem()
        base_item['table_name'] = 'jz_info'
        base_item['name'] = name
        base_item['social_code'] = info[0]
        base_item['legal_re'] = info[1]
        base_item['com_type'] = info[2]
        base_item['province'] = info[3]
        base_item['addr'] = info[4]
        yield base_item

        urls_ = response.xpath('//ul[@class="tinyTab datas_tabs"]/li/a/@data-url').extract()
        urls = {
            'credential_url_': urls_[0],
            'person_url_': urls_[1],
            'progress_url_': urls_[2],
            'g_behavior_url_': urls_[3],
            'b_behavior_url_': urls_[4],
            'blacklist_url_': urls_[5],
        }

        credential_url_ = urls_[0]
        credential_url = response.urljoin(credential_url_)

        cookies = eval(response.headers['cookies']) #这行的cookies是中间件写进去的
        yield Request(credential_url, callback=self.get_credential,
                      meta={'social_code': info[0], 'urls': urls, 'cookies':cookies},
                      cookies=cookies, dont_filter=True)

中间件写了一个cookie存入, 因为redis settings.py中没有打开cookie选项

除此之外，还有随机user-agent的中间件

class JszcDownloaderUserAgentMiddleware(UserAgentMiddleware):

    def __init__(self, user_agent=''):
        super(JszcDownloaderUserAgentMiddleware, self).__init__(user_agent)

    def process_request(self, request, spider):
        if not request.headers.get('User-Agent'):
            user_agent = choice(USER_AGENTS)
            request.headers.setdefault(b'User-Agent', user_agent)


class JszcDownloaderAutoCookieMiddleware():

    def process_response(self, request, response, spider):
        cookies =  response.headers.get('Set-Cookie')
        if cookies:
            l = filter(lambda x: '=' in x, cookies.decode('utf8').split(';') )
            coo = {}
            for i in l:
                s = i.split('=')
                coo[s[0]] = s[1]

            response.headers['cookies'] = str(coo)

        return response

运行

---

#cmd中
scrapy crawl getid 
scrapy crawl baseinfo

#redis-cli中
lpush getidspider:start_urls http://jzsc.mohurd.gov.cn/dataservice/query/comp/list

*记得先运行sql文件创建库

参考:

入库方法: 静觅 | Scrapy小技巧-MySQL存储