scrapy settings

1 scrapy settings　　

# -*- coding: utf-8 -*-

# Scrapy settings for phone project

BOT_NAME = 'phone'

SPIDER_MODULES = ['phone.spiders']
NEWSPIDER_MODULE = 'phone.spiders'


ROBOTSTXT_OBEY = False

DOWNLOAD_DELAY = 3

COOKIES_ENABLED = False

SPIDER_MIDDLEWARES = {
    # 'phone.middlewares.PhoneSpiderMiddleware': 543,
}

DOWNLOADER_MIDDLEWARES = {
    # 'phone.middlewares.MyCustomDownloaderMiddleware': 543,
}

ITEM_PIPELINES = {
    # 'phone.pipelines.PhonePipeline': 300,
}


SCHEDULER = "scrapy_redis.scheduler.Scheduler"
DUPEFILTER_CLASS = "scrapy_redis.dupefilter.RFPDupeFilter"
REDIS_URL = 'redis://root:123456@58.221.49.25:6379'