zoukankan      html  css  js  c++  java
  • Python 线程池的实现

    1. import urllib2  
    2. import time  
    3. import socket  
    4. from datetime import datetime  
    5. from thread_pool import *  
    6.    
    7. def main():  
    8.     url_list = {"sina":"http://www.sina.com.cn",  
    9.                "sohu":"http://www.sohu.com",  
    10.                "yahoo":"http://www.yahoo.com",  
    11.                "xiaonei":"http://www.xiaonei.com",  
    12.                "qihoo":"http://www.qihoo.com",  
    13.                "laohan":"http://www.laohan.org",  
    14.                "eyou":"http://www.eyou.com",  
    15.                "chinaren":"http://www.chinaren.com",  
    16.                "douban":"http://www.douban.com",  
    17.                "163":"http://www.163.com",  
    18.                "daqi":"http://www.daqi.com",  
    19.                "qq":"http://www.qq.com",  
    20.                "baidu_1":"http://www.baidu.com/s?wd=asdfasdf",  
    21.                "baidu_2":"http://www.baidu.com/s?wd=dddddddf",  
    22.                "google_1":"http://www.baidu.com/s?wd=sadfas",  
    23.                "google_2":"http://www.baidu.com/s?wd=sadflasd",  
    24.                "hainei":"http://www.hainei.com",  
    25.                "microsoft":"http://www.microsoft.com",  
    26.                "wlzuojia":"http://www.wlzuojia.com"}  
    27.    
    28.     #使用线程池  
    29.     socket.setdefaulttimeout(10)  
    30.     print 'start testing'  
    31.     wm = WorkerManager(50)  
    32.     for url_name in url_list.keys():  
    33.         wm.add_job(do_get_con, url_name, url_list[url_name])  
    34.     wm.wait_for_complete()  
    35.     print 'end testing'  
    36.    
    37. def do_get_con(url_name,url_link):  
    38.     try:  
    39.         fd = urllib2.urlopen(url_link)  
    40.         data = fd.read()  
    41.         f_hand = open("/tmp/ttt/%s" % url_name,"w")  
    42.         f_hand.write(data)  
    43.         f_hand.close()  
    44.     except Exception,e:  
    45.         pass  
    46.    
    47. if __name__ == "__main__":  
    48.     main()  
    49. thread_pool的代码(非原创,转自:http://blog.daviesliu.net/2006/10/09/234822/)  
    50.   
    51. import Queue, threading, sys  
    52. from threading import Thread  
    53. import time  
    54. import urllib  
    55.    
    56. # working thread  
    57. class Worker(Thread):  
    58.     worker_count = 0  
    59.     timeout = 1  
    60.     def __init__( self, workQueue, resultQueue, **kwds):  
    61.         Thread.__init__( self, **kwds )  
    62.         self.id = Worker.worker_count  
    63.         Worker.worker_count += 1  
    64.         self.setDaemon( True )  
    65.         self.workQueue = workQueue  
    66.         self.resultQueue = resultQueue  
    67.         self.start( )  
    68.    
    69.     def run( self ):  
    70.         ''''' the get-some-work, do-some-work main loop of worker threads '''  
    71.         while True:  
    72.             try:  
    73.                 callable, args, kwds = self.workQueue.get(timeout=Worker.timeout)  
    74.                 res = callable(*args, **kwds)  
    75.                 print "worker[%2d]: %s" % (self.id, str(res) )  
    76.                 self.resultQueue.put( res )  
    77.                 #time.sleep(Worker.sleep)  
    78.             except Queue.Empty:  
    79.                 break  
    80.             except :  
    81.                 print 'worker[%2d]' % self.id, sys.exc_info()[:2]  
    82.                 raise  
    83.                  
    84. class WorkerManager:  
    85.     def __init__( self, num_of_workers=10, timeout = 2):  
    86.         self.workQueue = Queue.Queue()  
    87.         self.resultQueue = Queue.Queue()  
    88.         self.workers = []  
    89.         self.timeout = timeout  
    90.         self._recruitThreads( num_of_workers )  
    91.    
    92.     def _recruitThreads( self, num_of_workers ):  
    93.         for i in range( num_of_workers ):  
    94.             worker = Worker( self.workQueue, self.resultQueue )  
    95.             self.workers.append(worker)  
    96.    
    97.     def wait_for_complete( self):  
    98.         # ...then, wait for each of them to terminate:  
    99.         while len(self.workers):  
    100.             worker = self.workers.pop()  
    101.             worker.join( )  
    102.             if worker.isAlive() and not self.workQueue.empty():  
    103.                 self.workers.append( worker )  
    104.         print "All jobs are are completed."  
    105.    
    106.     def add_job( self, callable, *args, **kwds ):  
    107.         self.workQueue.put( (callable, args, kwds) )  
    108.    
    109.     def get_result( self, *args, **kwds ):  
    110.         return self.resultQueue.get( *args, **kwds )  
  • 相关阅读:
    C# 从服务器下载文件
    不能使用联机NuGet 程序包
    NPOI之Excel——合并单元格、设置样式、输入公式
    jquery hover事件中 fadeIn和fadeOut 效果不能及时停止
    UVA 10519 !! Really Strange !!
    UVA 10359 Tiling
    UVA 10940 Throwing cards away II
    UVA 10079 Pizze Cutting
    UVA 763 Fibinary Numbers
    UVA 10229 Modular Fibonacci
  • 原文地址:https://www.cnblogs.com/skying555/p/4499565.html
Copyright © 2011-2022 走看看