zoukankan      html  css  js  c++  java
  • Python并发请求之requests_future模块使用

    # -*- coding: utf-8 -*-
    # @Time : 2019-12-09 10:00
    # @Author : cxa
    # @File : demo.py
    # @Software: PyCharm
    from requests_futures.sessions import FuturesSession
    from concurrent.futures import as_completed
    from lxml import html
    import time
    
    url = ["http://www.baidu.com", "http://www.163.com", "http://www.google.com", "http://www.cnblogs.com/c-x-a"]
    
    
    def get_node(source, x=".//head/title//text()"):
        root = html.fromstring(source)
        node = root.xpath(x)
        return node
    
    
    def response_hook(resp, *args, **kwargs):
        start = time.time()
        resp.encoding = resp.apparent_encoding
        resp.data = resp.text
        resp.code = resp.status_code
        resp.headers = resp.headers
        resp.elapsed = time.time() - start
    
    
    def get_req():
        with FuturesSession(max_workers=4) as session:
            futures = [session.get(i, hooks={"response": response_hook}) for i in url]
            for future in as_completed(futures):
                resp = future.result()
                print("状态码", resp.code)
                print("标题", get_node(resp.data)[0])
                print("耗时", resp.elapsed)
                print("="*30)
    
    if __name__ == '__main__':
        get_req()
    
    

    修改之后

    from requests_futures.sessions import FuturesSession
    from concurrent.futures import as_completed
    from lxml import html
    import time
    
    url = ["http://www.baidu.com", "http://www.163.com", "http://www.google.com", "http://www.cnblogs.com/c-x-a"]
    
    
    class MySession(FuturesSession):
    
        def request(self, method, url, hooks=None, *args, **kwargs):
            start = time.time()
            if hooks is None:
                hooks = {}
    
            def response_hook(resp, *args, **kwargs):
                resp.encoding = resp.apparent_encoding
                resp.data = resp.text
                resp.code = resp.status_code
                resp.headers = resp.headers
                resp.elapsed = time.time() - start
    
            try:
                if isinstance(hooks['response'], (list, tuple)):
                    hooks['response'].insert(0, response_hook)
                else:
                    hooks['response'] = [response_hook, hooks['response']]
            except KeyError:
                hooks['response'] = response_hook
    
            return super(MySession, self).request(method, url, hooks=hooks, *args, **kwargs)
    
    
    def get_node(source, x=".//head/title//text()"):
        root = html.fromstring(source)
        node = root.xpath(x)
        return node
    
    
    def get_req():
        with MySession(max_workers=4) as session:
            futures = [session.get(i) for i in url]
            for future in as_completed(futures):
                resp = future.result()
                print("状态码", resp.code)
                print("标题", get_node(resp.data)[0])
                print("耗时", resp.elapsed)
                print("=" * 30)
    
    
    if __name__ == '__main__':
        get_req()
    
  • 相关阅读:
    sql经典语句大全
    经典SQL语句大全
    Bat命令学习
    [Microsoft][ODBC 驱动程序管理器] 在指定的 DSN 中,驱动程序和应用程序之间的体系结构不匹配
    配置WebSite的IIS时遇到的问题与解决方法
    数据库SQL优化大总结之 百万级数据库优化方案
    数据库索引以及优化
    搭建android开发环境
    SQL2008根据日志恢复
    WebService处理大数据量数据
  • 原文地址:https://www.cnblogs.com/c-x-a/p/12009565.html
Copyright © 2011-2022 走看看