1 # scheduler的作用: 用于控制Request对象的存储和获取，并提供了过滤重复Request的功能。 2 3 class Scheduler(object): 4 5 def __init__(self, dupefilter, jobdir=None, dqclass=None, mqclass=None, 6 logunser=False, stats=None, pqclass=None): 7 self.df = dupefilter 8 self.dqdir = self._dqdir(jobdir) 9 self.pqclass = pqclass # 优先级队列 10 self.dqclass = dqclass # 磁盘序列号队列,用于断点续采 11 self.mqclass = mqclass # 内存队列 12 self.logunser = logunser 13 self.stats = stats 14 15 @classmethod 16 def from_crawler(cls, crawler): 17 settings = crawler.settings 18 dupefilter_cls = load_object(settings['DUPEFILTER_CLASS']) 19 dupefilter = dupefilter_cls.from_settings(settings) 20 pqclass = load_object(settings['SCHEDULER_PRIORITY_QUEUE']) 21 dqclass = load_object(settings['SCHEDULER_DISK_QUEUE']) 22 mqclass = load_object(settings['SCHEDULER_MEMORY_QUEUE']) 23 logunser = settings.getbool('LOG_UNSERIALIZABLE_REQUESTS', settings.getbool('SCHEDULER_DEBUG')) 24 return cls(dupefilter, jobdir=job_dir(settings), logunser=logunser, 25 stats=crawler.stats, pqclass=pqclass, dqclass=dqclass, mqclass=mqclass)