zoukankan      html  css  js  c++  java
  • pool=eventlet.GreenPool(20)

    默认是1000,可能是我在 ubuntu下unlimit -n没有调 ,或是python的httplib有问题,抓同一个站就抱错


        r = h.getresponse()
      File "/usr/lib/python2.6/httplib.py", line 986, in getresponse
        response.begin()
      File "/usr/lib/python2.6/httplib.py", line 391, in begin
        version, status, reason = self._read_status()
      File "/usr/lib/python2.6/httplib.py", line 355, in _read_status
        raise BadStatusLine(line)
    BadStatusLine
    Traceback (most recent call last):
      File "/usr/local/lib/python2.6/dist-packages/eventlet-0.9.9-py2.6.egg/eventlet/greenpool.py", line 80, in _spawn_n_impl
        func(*args, **kwargs)
      File "batch_download.py", line 17, in fetch
        html=urllib2.urlopen(url).read()
      File "/usr/lib/python2.6/urllib2.py", line 126, in urlopen
        return _opener.open(url, data, timeout)
      File "/usr/lib/python2.6/urllib2.py", line 391, in open
        response = self._open(req, data)
      File "/usr/lib/python2.6/urllib2.py", line 409, in _open
        '_open', req)
      File "/usr/lib/python2.6/urllib2.py", line 369, in _call_chain
        result = func(*args)
      File "/usr/lib/python2.6/urllib2.py", line 1161, in http_open
        return self.do_open(httplib.HTTPConnection, req)
      File "/usr/lib/python2.6/urllib2.py", line 1134, in do_open
        r = h.getresponse()
      File "/usr/lib/python2.6/httplib.py", line 986, in getresponse
        response.begin()
      File "/usr/lib/python2.6/httplib.py", line 391, in begin
        version, status, reason = self._read_status()
      File "/usr/lib/python2.6/httplib.py", line 355, in _read_status
        raise BadStatusLine(line)
    BadStatusLine
    Traceback (most recent call last):
      File "/usr/local/lib/python2.6/dist-packages/eventlet-0.9.9-py2.6.egg/eventlet/greenpool.py", line 80, in _spawn_n_impl
        func(*args, **kwargs)
      File "batch_download.py", line 17, in fetch
        html=urllib2.urlopen(url).read()
      File "/usr/lib/python2.6/urllib2.py", line 126, in urlopen
        return _opener.open(url, data, timeout)
      File "/usr/lib/python2.6/urllib2.py", line 391, in open
        response = self._open(req, data)
      File "/usr/lib/python2.6/urllib2.py", line 409, in _open
        '_open', req)
      File "/usr/lib/python2.6/urllib2.py", line 369, in _call_chain
        result = func(*args)
      File "/usr/lib/python2.6/urllib2.py", line 1161, in http_open
        return self.do_open(httplib.HTTPConnection, req)
      File "/usr/lib/python2.6/urllib2.py", line 1134, in do_open
        r = h.getresponse()
      File "/usr/lib/python2.6/httplib.py", line 986, in getresponse
        response.begin()
      File "/usr/lib/python2.6/httplib.py", line 391, in begin
        version, status, reason = self._read_status()
      File "/usr/lib/python2.6/httplib.py", line 355, in _read_status
        raise BadStatusLine(line)
    BadStatusLine
    Traceback (most recent call last):
      File "/usr/local/lib/python2.6/dist-packages/eventlet-0.9.9-py2.6.egg/eventlet/greenpool.py", line 80, in _spawn_n_impl
        func(*args, **kwargs)
      File "batch_download.py", line 17, in fetch
        html=urllib2.urlopen(url).read()
      File "/usr/lib/python2.6/urllib2.py", line 126, in urlopen
        return _opener.open(url, data, timeout)
      File "/usr/lib/python2.6/urllib2.py", line 391, in open
        response = self._open(req, data)
      File "/usr/lib/python2.6/urllib2.py", line 409, in _open
        '_open', req)
      File "/usr/lib/python2.6/urllib2.py", line 369, in _call_chain
        result = func(*args)
      File "/usr/lib/python2.6/urllib2.py", line 1161, in http_open
        return self.do_open(httplib.HTTPConnection, req)
      File "/usr/lib/python2.6/urllib2.py", line 1134, in do_open
        r = h.getresponse()
      File "/usr/lib/python2.6/httplib.py", line 986, in getresponse
        response.begin()
      File "/usr/lib/python2.6/httplib.py", line 391, in begin
        version, status, reason = self._read_status()
      File "/usr/lib/python2.6/httplib.py", line 355, in _read_status
        raise BadStatusLine(line)
    BadStatusLine
    Traceback (most recent call last):
      File "/usr/local/lib/python2.6/dist-packages/eventlet-0.9.9-py2.6.egg/eventlet/greenpool.py", line 80, in _spawn_n_impl
        func(*args, **kwargs)
      File "batch_download.py", line 17, in fetch
        html=urllib2.urlopen(url).read()
      File "/usr/lib/python2.6/urllib2.py", line 126, in urlopen
        return _opener.open(url, data, timeout)
      File "/usr/lib/python2.6/urllib2.py", line 391, in open
        response = self._open(req, data)
      File "/usr/lib/python2.6/urllib2.py", line 409, in _open
        '_open', req)
      File "/usr/lib/python2.6/urllib2.py", line 369, in _call_chain
        result = func(*args)
      File "/usr/lib/python2.6/urllib2.py", line 1161, in http_open
        return self.do_open(httplib.HTTPConnection, req)
      File "/usr/lib/python2.6/urllib2.py", line 1134, in do_open
        r = h.getresponse()
      File "/usr/lib/python2.6/httplib.py", line 986, in getresponse
        response.begin()
      File "/usr/lib/python2.6/httplib.py", line 391, in begin
        version, status, reason = self._read_status()
      File "/usr/lib/python2.6/httplib.py", line 355, in _read_status
        raise BadStatusLine(line)
    BadStatusLine
    Traceback (most recent call last):
      File "/usr/local/lib/python2.6/dist-packages/eventlet-0.9.9-py2.6.egg/eventlet/greenpool.py", line 80, in _spawn_n_impl
        func(*args, **kwargs)
      File "batch_download.py", line 17, in fetch
        html=urllib2.urlopen(url).read()
      File "/usr/lib/python2.6/urllib2.py", line 126, in urlopen
        return _opener.open(url, data, timeout)
      File "/usr/lib/python2.6/urllib2.py", line 391, in open
        response = self._open(req, data)
      File "/usr/lib/python2.6/urllib2.py", line 409, in _open
        '_open', req)
      File "/usr/lib/python2.6/urllib2.py", line 369, in _call_chain
        result = func(*args)
      File "/usr/lib/python2.6/urllib2.py", line 1161, in http_open
        return self.do_open(httplib.HTTPConnection, req)
      File "/usr/lib/python2.6/urllib2.py", line 1134, in do_open
        r = h.getresponse()
      File "/usr/lib/python2.6/httplib.py", line 986, in getresponse
        response.begin()
      File "/usr/lib/python2.6/httplib.py", line 391, in begin
        version, status, reason = self._read_status()
      File "/usr/lib/python2.6/httplib.py", line 355, in _read_status
        raise BadStatusLine(line)
    BadStatusLine
    Traceback (most recent call last):
      File "/usr/local/lib/python2.6/dist-packages/eventlet-0.9.9-py2.6.egg/eventlet/greenpool.py", line 80, in _spawn_n_impl
        func(*args, **kwargs)
      File "batch_download.py", line 17, in fetch
        html=urllib2.urlopen(url).read()
      File "/usr/lib/python2.6/urllib2.py", line 126, in urlopen
        return _opener.open(url, data, timeout)
      File "/usr/lib/python2.6/urllib2.py", line 391, in open
        response = self._open(req, data)
      File "/usr/lib/python2.6/urllib2.py", line 409, in _open
        '_open', req)
      File "/usr/lib/python2.6/urllib2.py", line 369, in _call_chain
        result = func(*args)
      File "/usr/lib/python2.6/urllib2.py", line 1161, in http_open
        return self.do_open(httplib.HTTPConnection, req)
      File "/usr/lib/python2.6/urllib2.py", line 1134, in do_open
        r = h.getresponse()
      File "/usr/lib/python2.6/httplib.py", line 986, in getresponse
        response.begin()
      File "/usr/lib/python2.6/httplib.py", line 391, in begin
        version, status, reason = self._read_status()
      File "/usr/lib/python2.6/httplib.py", line 355, in _read_status
        raise BadStatusLine(line)
    BadStatusLine
    Traceback (most recent call last):
      File "/usr/local/lib/python2.6/dist-packages/eventlet-0.9.9-py2.6.egg/eventlet/greenpool.py", line 80, in _spawn_n_impl
        func(*args, **kwargs)
      File "batch_download.py", line 17, in fetch
        html=urllib2.urlopen(url).read()
      File "/usr/lib/python2.6/urllib2.py", line 126, in urlopen
        return _opener.open(url, data, timeout)
      File "/usr/lib/python2.6/urllib2.py", line 391, in open
        response = self._open(req, data)
      File "/usr/lib/python2.6/urllib2.py", line 409, in _open
        '_open', req)
      File "/usr/lib/python2.6/urllib2.py", line 369, in _call_chain
        result = func(*args)
      File "/usr/lib/python2.6/urllib2.py", line 1161, in http_open
        return self.do_open(httplib.HTTPConnection, req)
      File "/usr/lib/python2.6/urllib2.py", line 1134, in do_open
        r = h.getresponse()
      File "/usr/lib/python2.6/httplib.py", line 986, in getresponse
        response.begin()
      File "/usr/lib/python2.6/httplib.py", line 391, in begin
        version, status, reason = self._read_status()
      File "/usr/lib/python2.6/httplib.py", line 355, in _read_status
        raise BadStatusLine(line)
    BadStatusLine
    Traceback (most recent call last):
      File "/usr/local/lib/python2.6/dist-packages/eventlet-0.9.9-py2.6.egg/eventlet/greenpool.py", line 80, in _spawn_n_impl
        func(*args, **kwargs)
      File "batch_download.py", line 17, in fetch
        html=urllib2.urlopen(url).read()
      File "/usr/lib/python2.6/urllib2.py", line 126, in urlopen
        return _opener.open(url, data, timeout)
      File "/usr/lib/python2.6/urllib2.py", line 391, in open
        response = self._open(req, data)
      File "/usr/lib/python2.6/urllib2.py", line 409, in _open
        '_open', req)
      File "/usr/lib/python2.6/urllib2.py", line 369, in _call_chain
        result = func(*args)
      File "/usr/lib/python2.6/urllib2.py", line 1161, in http_open
        return self.do_open(httplib.HTTPConnection, req)
      File "/usr/lib/python2.6/urllib2.py", line 1134, in do_open
        r = h.getresponse()
      File "/usr/lib/python2.6/httplib.py", line 986, in getresponse
        response.begin()
      File "/usr/lib/python2.6/httplib.py", line 391, in begin
        version, status, reason = self._read_status()
      File "/usr/lib/python2.6/httplib.py", line 355, in _read_status
        raise BadStatusLine(line)
    BadStatusLine
    Traceback (most recent call last):
      File "/usr/local/lib/python2.6/dist-packages/eventlet-0.9.9-py2.6.egg/eventlet/greenpool.py", line 80, in _spawn_n_impl
        func(*args, **kwargs)
      File "batch_download.py", line 17, in fetch
        html=urllib2.urlopen(url).read()
      File "/usr/lib/python2.6/urllib2.py", line 126, in urlopen
        return _opener.open(url, data, timeout)
      File "/usr/lib/python2.6/urllib2.py", line 391, in open
        response = self._open(req, data)
      File "/usr/lib/python2.6/urllib2.py", line 409, in _open
        '_open', req)
      File "/usr/lib/python2.6/urllib2.py", line 369, in _call_chain
        result = func(*args)
      File "/usr/lib/python2.6/urllib2.py", line 1161, in http_open
        return self.do_open(httplib.HTTPConnection, req)
      File "/usr/lib/python2.6/urllib2.py", line 1134, in do_open
        r = h.getresponse()
      File "/usr/lib/python2.6/httplib.py", line 986, in getresponse
        response.begin()
      File "/usr/lib/python2.6/httplib.py", line 391, in begin
        version, status, reason = self._read_status()
      File "/usr/lib/python2.6/httplib.py", line 355, in _read_status
        raise BadStatusLine(line)
    BadStatusLine
    ^CTraceback (most recent call last):
      File "batch_download.py", line 29, in <module>
        pool.spawn_n(fetch, url)
      File "/usr/local/lib/python2.6/dist-packages/eventlet-0.9.9-py2.6.egg/eventlet/greenpool.py", line 104, in spawn_n
        self.sem.acquire()
      File "/usr/local/lib/python2.6/dist-packages/eventlet-0.9.9-py2.6.egg/eventlet/semaphore.py", line 71, in acquire
        hubs.get_hub().switch()
      File "/usr/local/lib/python2.6/dist-packages/eventlet-0.9.9-py2.6.egg/eventlet/hubs/hub.py", line 135, in switch
        return self.greenlet.switch()
      File "/usr/local/lib/python2.6/dist-packages/eventlet-0.9.9-py2.6.egg/eventlet/hubs/hub.py", line 169, in run
        self.fire_timers(self.clock())
      File "/usr/local/lib/python2.6/dist-packages/eventlet-0.9.9-py2.6.egg/eventlet/hubs/hub.py", line 285, in fire_timers
        timer()
      File "/usr/local/lib/python2.6/dist-packages/eventlet-0.9.9-py2.6.egg/eventlet/hubs/timer.py", line 56, in __call__
        cb(*args, **kw)
      File "/usr/local/lib/python2.6/dist-packages/eventlet-0.9.9-py2.6.egg/eventlet/greenpool.py", line 80, in _spawn_n_impl
        func(*args, **kwargs)
      File "batch_download.py", line 17, in fetch
        html=urllib2.urlopen(url).read()
      File "/usr/lib/python2.6/urllib2.py", line 126, in urlopen
        return _opener.open(url, data, timeout)
      File "/usr/lib/python2.6/urllib2.py", line 391, in open
        response = self._open(req, data)
      File "/usr/lib/python2.6/urllib2.py", line 409, in _open
        '_open', req)
      File "/usr/lib/python2.6/urllib2.py", line 369, in _call_chain
        result = func(*args)
      File "/usr/lib/python2.6/urllib2.py", line 1161, in http_open
        return self.do_open(httplib.HTTPConnection, req)
      File "/usr/lib/python2.6/urllib2.py", line 1133, in do_open
        h.request(req.get_method(), req.get_selector(), req.data, headers)
      File "/usr/lib/python2.6/httplib.py", line 910, in request
    ^[[A    self._send_request(method, url, body, headers)
      File "/usr/lib/python2.6/httplib.py", line 947, in _send_request
        self.endheaders()
      File "/usr/lib/python2.6/httplib.py", line 904, in endheaders
        self._send_output()
      File "/usr/lib/python2.6/httplib.py", line 776, in _send_output
        self.send(msg)
      File "/usr/lib/python2.6/httplib.py", line 735, in send
        self.connect()
      File "/usr/lib/python2.6/httplib.py", line 716, in connect
        self.timeout)
      File "/usr/local/lib/python2.6/dist-packages/eventlet-0.9.9-py2.6.egg/eventlet/green/socket.py", line 78, in create_connection
        for res in getaddrinfo(host, port, 0, SOCK_STREAM):
    KeyboardInterrupt
    mlzboy@mlzboy-mac:~/my/idea2/download/hqs$ python batch_download.py ^C
    mlzboy@mlzboy-mac:~/my/idea2/download/hqs$ python batch_download.py
    Traceback (most recent call last):
      File "/usr/local/lib/python2.6/dist-packages/eventlet-0.9.9-py2.6.egg/eventlet/greenpool.py", line 80, in _spawn_n_impl
        func(*args, **kwargs)
      File "batch_download.py", line 17, in fetch
        html=urllib2.urlopen(url).read()
      File "/usr/lib/python2.6/urllib2.py", line 126, in urlopen
        return _opener.open(url, data, timeout)
      File "/usr/lib/python2.6/urllib2.py", line 391, in open
        response = self._open(req, data)
      File "/usr/lib/python2.6/urllib2.py", line 409, in _open
        '_open', req)
      File "/usr/lib/python2.6/urllib2.py", line 369, in _call_chain
        result = func(*args)
      File "/usr/lib/python2.6/urllib2.py", line 1161, in http_open
        return self.do_open(httplib.HTTPConnection, req)
      File "/usr/lib/python2.6/urllib2.py", line 1136, in do_open
        raise URLError(err)
    URLError: <urlopen error [Errno 110] ETIMEDOUT>

    刚发现设pool的一个好处是一个坏了,不影响其它的,只是速度慢一点,对我而言,少一两个页面也没事,



  • 相关阅读:
    禁用生成8.3字符长文件名解决单目录存储大量文件写入性能下降问题
    遭遇EventSystem服务问题
    软件工程师遭遇硬件故障RAID5两块硬盘离线
    Pocket Player睡眠定时器不能选择问题的解决办法
    借助RamDisk让你的FireFox飞起来
    NVS4200M显卡“超频”
    Python数据类型
    Python变量
    端口及对应服务
    Python基础(1)
  • 原文地址:https://www.cnblogs.com/lexus/p/1787041.html
Copyright © 2011-2022 走看看