zoukankan      html  css  js  c++  java
  • aiohttp/asyncio 多次请求

    #!/usr/bin/env python
    # -*- coding: utf-8 -*-
    
    __author__ = "Daniel Altiparmak (sixfinger78@gmail.com)"
    __copyright__ = "Copyright (C) 2015 Daniel Altiparmak"
    __license__ = "GPL 3.0"
    
    
    import asyncio
    import aiohttp
    import tqdm
    
    import string
    import random
    
    # get content and write it to file
    def write_to_file(filename, content):
        f = open(filename, 'wb')
        f.write(content)
        f.close()
    
    # a helper coroutine to perform GET requests:
    @asyncio.coroutine
    def get(*args, **kwargs):
        response = yield from aiohttp.request('GET', *args, **kwargs)
        return (yield from response.read_and_close())
    
    
    @asyncio.coroutine
    def download_file(url):
        # this routine is protected by a semaphore
        with (yield from r_semaphore):
            content = yield from asyncio.async(get(url))
    
            # create random filename
            length = 10
            file_string = ''.join(random.choice(
                string.ascii_lowercase + string.digits) for _ in range(length)
                                  )
            filename = '{}.png'.format(file_string)
    
            write_to_file(filename, content)
    
    '''
    make nice progressbar
    install it by using `pip install tqdm`
    '''
    @asyncio.coroutine
    def wait_with_progressbar(coros):
        for f in tqdm.tqdm(asyncio.as_completed(coros), total=len(coros)):
            yield from f
    
    
    images = ['http://lorempixel.com/1920/1920/' for i in range(100)]
    
    
    # avoid to many requests(coroutines) the same time.
    # limit them by setting semaphores (simultaneous requests)
    r_semaphore = asyncio.Semaphore(10)
    
    coroutines = [download_file(url) for url in images]
    eloop = asyncio.get_event_loop()
    #eloop.run_until_complete(asyncio.wait(coroutines))
    eloop.run_until_complete(wait_with_progressbar(coroutines))
    eloop.close()
    

      

  • 相关阅读:
    编译i386 Linux 内核并基于 QEMU 运行
    在 Linux 上编译运行并测试 LwIP 协议栈性能
    gdb 重定位源文件目录
    SkyWalking 分布式追踪系统
    微服务化的基石——持续集成(二)
    微服务容器化的分工与合作,促进DevOps (一)
    预订餐位
    单词记录1.26
    What do you do on weekends
    make a travel plan(LC)
  • 原文地址:https://www.cnblogs.com/c-x-a/p/9025588.html
Copyright © 2011-2022 走看看