用 python 实现一个多线程网页下载器

很久没有更新博客了，今天上来分享一下昨天实现的一个多线程网页下载器。

这是一个有着真实需求的实现，我的用途是拿它来通过 HTTP 方式向服务器提交游戏数据。把它放上来也是想大家帮忙挑刺，找找 bug，让它工作得更好。

keywords:python,http,multi-threads,thread,threading,httplib,urllib,urllib2,Queue,http pool,httppool

废话少说，上源码：

# -*- coding:utf-8 -*- import urllib, httplib import thread import time from Queue import Queue, Empty, Full HEADERS = {"Content-type": "application/x-www-form-urlencoded", 'Accept-Language':'zh-cn', 'User-Agent': 'Mozilla/4.0 (compatible; MSIE 6.0;Windows NT 5.0)', "Accept": "text/plain"} UNEXPECTED_ERROR = -1 POST = 'POST' GET = 'GET' def base_log(msg): print msg def base_fail_op(task, status, log): log('fail op. task = %s, status = %d'%(str(task), status)) def get_remote_data(tasks, results, fail_op = base_fail_op, log = base_log): while True: task = tasks.get() try: tid = task['id'] hpt = task['conn_args'] # hpt <= host:port, timeout except KeyError, e: log(str(e)) continue log('thread_%s doing task %d'%(thread.get_ident(), tid)) #log('hpt = ' + str(hpt)) conn = httplib.HTTPConnection(**hpt) try: params = task['params'] except KeyError, e: params = {} params = urllib.urlencode(params) #log('params = ' + params) try: method = task['method'] except KeyError: method = 'GET' #log('method = ' + method) try: url = task['url'] except KeyError: url = '/' #log('url = ' + url) headers = HEADERS try: tmp = task['headers'] except KeyError, e: tmp = {} headers.update(tmp) #log('headers = ' + str(headers)) headers['Content-Length'] = len(params) try: if method == POST: conn.request(method, url, params, headers) else: conn.request(method, url + params) response = conn.getresponse() except Exception, e: log('request failed. method = %s, url = %s, params = %s headers = %s'%( method, url, params, headers)) log(str(e)) fail_op(task, UNEXPECTED_ERROR, log) continue if response.status != httplib.OK: fail_op(task, response.status, log) continue data = response.read() results.put((tid, data), True) class HttpPool(object): def __init__(self, threads_count, fail_op, log): self._tasks = Queue() self._results = Queue() for i in xrange(threads_count): thread.start_new_thread(get_remote_data, (self._tasks, self._results, fail_op, log)) def add_task(self, tid, host, url, params, headers = {}, method = 'GET', timeout = None): task = { 'id' : tid, 'conn_args' : {'host' : host} if timeout is None else {'host' : host, 'timeout' : timeout}, 'headers' : headers, 'url' : url, 'params' : params, 'method' : method, } try: self._tasks.put_nowait(task) except Full: return False return True def get_results(self): results = [] while True: try: res = self._results.get_nowait() except Empty: break results.append(res) return results def test_google(task_count, threads_count): hp = HttpPool(threads_count, base_fail_op, base_log) for i in xrange(task_count): if hp.add_task(i, 'www.google.cn', '/search?', {'q' : 'lai'}, # method = 'POST' ): print 'add task successed.' while True: results = hp.get_results() if not results: time.sleep(1.0 * random.random()) for i in results: print i[0], len(i[1]) # print unicode(i[1], 'gb18030') if __name__ == '__main__': import sys, random task_count, threads_count = int(sys.argv[1]), int(sys.argv[2]) test_google(task_count, threads_count)

有兴趣想尝试运行的朋友，可以把它保存为 xxxx.py，然后执行 python xxxx.py 10 4，其中 10 表示向 google.cn 请求 10 次查询，4 表示由 4 条线程来执行这些任务。