zoukankan      html  css  js  c++  java
  • tornado实现高并发爬虫

    from urllib.parse import urljoin
    
    from bs4 import BeautifulSoup
    from tornado import gen, httpclient, ioloop, queues
    
    base_url = "http://www.tornadoweb.org/en/stable/"
    concurrency = 3
    
    async def get_url_links(url):
        response = await httpclient.AsyncHTTPClient().fetch("http://www.tornadoweb.org/en/stable/")
        html = response.body.decode("utf8")
        soup = BeautifulSoup(html)
        links = [urljoin(base_url, a.get("href")) for a in soup.find_all("a", href=True)]
        return links
    
    async def main():
        seen_set = set()
        q = queues.Queue()
    
        async def fetch_url(current_url):
            #生产者
            if current_url in seen_set:
                return
    
            print("获取: {}".format(current_url))
            seen_set.add(current_url)
            next_urls = await get_url_links(current_url)
            for new_url in next_urls:
                if new_url.startswith(base_url):
                    #放入队列,
                    await q.put(new_url)
    
        async def worker():
            async for url in q:
                if url is None:
                    return
                try:
                    await fetch_url(url)
                except Exception as e:
                    print("excepiton")
                finally:
                    q.task_done()
    
        #放入初始url到队列
        await q.put(base_url)
    
        #启动协程
        workers = gen.multi([worker() for _ in range(concurrency)])
        await q.join()
    
        for _ in range(concurrency):
            await q.put(None)
    
        await workers
    
    
    if __name__ == "__main__":
        io_loop = ioloop.IOLoop.current()
        io_loop.run_sync(main)
  • 相关阅读:
    DS博客作业02--栈和队列
    DS博客作业01-线性表
    c博客06-结构体&文件
    C博客作业05--2019-指针
    C语言博客作业04——数组
    C语言博客作业03--函数
    面向对象设计大作业第二阶段:图书馆系统
    JAVA程序设计-有理数类的设计
    DS博客作业05--查找
    DS博客作业04--图
  • 原文地址:https://www.cnblogs.com/topass123/p/13404773.html
Copyright © 2011-2022 走看看