zoukankan      html  css  js  c++  java
  • scrapy爬取西刺网站ip

    # scrapy爬取西刺网站ip
    # -*- coding: utf-8 -*-
    import scrapy
    
    from xici.items import XiciItem
    
    
    class XicispiderSpider(scrapy.Spider):
        name = "xicispider"
        allowed_domains = ["www.xicidaili.com/nn"]
        start_urls = ['http://www.xicidaili.com/nn/']
    
        def parse(self, response):
            item = XiciItem()
            for each in response.css('#ip_list tr'):
                ip = each.css('td:nth-child(2)::text').extract_first()
                port = each.css('td:nth-child(3)::text').extract_first()
                if ip:
                    ip_port = ip + ':' + port
                    item['ip_port'] = ip_port
                    yield item
    import pymongo
    
    class XiciPipeline(object):
    
        collection_name = 'scrapy_items'
    
        def __init__(self, mongo_uri, mongo_db):
            self.mongo_uri = mongo_uri
            self.mongo_db = mongo_db
        #这里的from经常拼错啊
        @classmethod
        def from_crawler(cls, crawler):
            return cls(
                mongo_uri=crawler.settings.get('MONGO_URI'),
                mongo_db=crawler.settings.get('MONGO_DB')
            )
    
        def open_spider(self, spider):
            self.client = pymongo.MongoClient(self.mongo_uri)
            self.db = self.client[self.mongo_db]
    
        def close_spider(self, spider):
            self.client.close()
    
        def process_item(self, item, spider):
            self.db[self.collection_name].insert(dict(item))
            return item
  • 相关阅读:
    centos7安装pycharm
    centos7 mysql数据库安装
    删除MySQL服务
    计组第三章预习
    攻防世界web新手练习区
    原码补码预习
    第一次总结
    第三章预习
    数据结构十进制数表示
    预习原码补码
  • 原文地址:https://www.cnblogs.com/themost/p/7110378.html
Copyright © 2011-2022 走看看