zoukankan      html  css  js  c++  java
  • 使用IO多路复用完成HTTP请求

    import socket
    from urllib.parse import urlparse
    from selectors import DefaultSelector, EVENT_READ, EVENT_WRITE
    
    
    selecter = DefaultSelector()
    urls = ["http://app2.jg.eastmoney.com/m/ScienceBoardTopic/index.html#/topic"]
    stop = False
    
    
    # 通过socket请求html
    # 使用select完成http请求
    # select + 回调 + 事件循环
    class Fetcher:
        def connected(self, key):
            selecter.unregister(key.fd)
            self.client.send("GET {} HTTP/1.1
    Host:{}
    Connection:close
    
    ".format(self.path, self.host).encode("utf8"))
            selecter.register(self.client.fileno(), EVENT_READ, self.readable)
    
        def readable(self, key):
            d = self.client.recv(1024)
            if d:
                self.data += d
            else:
                selecter.unregister(key.fd)
                data = self.data.decode("utf8")
                html_data = data.split("
    
    ")[1:]
                print("
    
    ".join(html_data))
                self.client.close()
                urls.remove(self.spider_url)
                if not urls:
                    global stop
                    stop = True
    
        def get_url(self, url):
            self.spider_url = url
            self.data = b""
            url = urlparse(url)
            self.host = url.netloc
            self.path = url.path
            if self.path == "":
                self.path = "/"
    
            # 建立socket连接
            self.client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            self.client.setblocking(False)
            try:
                self.client.connect((self.host, 80))
            except BlockingIOError as e:
                pass
    
            # 注册
            selecter.register(self.client.fileno(), EVENT_WRITE, self.connected)
    
    
    # 事件循环
    def loop():
        while not stop:
            ready = selecter.select()
            for key, mask in ready:
                call_back = key.data
                call_back(key)
    
    
    if __name__ == "__main__":
        fetcher = Fetcher()
        fetcher.get_url("http://www.baidu.com")
        loop()
  • 相关阅读:
    shell 指定范围产生随机数
    shell脚本比较两个数大小
    Shell 脚本实现随机抽取班级学生
    linux通过挂载系统光盘搭建本地yum仓库的方法
    kuberenetes 上使用helm部署rancher如何卸载干净
    Windows 下 左Ctrl和Caps交换
    C#笔记 -- 协变、逆变
    Python 读取window下UTF-8-BOM 文件
    生成命令行程序使用脚本
    ffmpeg 命令小记
  • 原文地址:https://www.cnblogs.com/yejing-snake/p/14263821.html
Copyright © 2011-2022 走看看