多线程

import time
from concurrent.futures import ThreadPoolExecutor, as_completed, ProcessPoolExecutor


class ThreadPoolSpider:
    executor = ThreadPoolExecutor(max_workers=8)
    # executor = ProcessPoolExecutor(max_workers=8)

    def __init__(self):
        pass

    def http_request(self, url, second):
        time.sleep(second)
        return url, second

    def crawl(self):
        all_tasks = []
        for index in range(100):
            task = self.executor.submit(self.http_request, index, 10)
            all_tasks.append(task)

        for result in as_completed(all_tasks):
            data = result.result()
            print(data)


if __name__ == '__main__':
    spider = ThreadPoolSpider()
    spider.crawl()

ThreadPoolExecutor：线程池
as_completed()方法是一个生成器，在没有任务完成的时候，会阻塞，在有某个任务完成的时候，会yield这个任务，就能执行for循环下面的语句，然后继续阻塞住，循环到所有的任务结束