python线程池实现多线程

参考文献

http://www.open-open.com/news/view/1c0179b

http://blog.jobbole.com/52060/

按照这个博客，实现获取多台服务器的空间使用情况

代码如下：

#!/usr/bin/env python2.7
#-*- coding:utf-8 -*-

from multiprocessing.dummy import Pool as ThreadPool
import subprocess
import time

results = []
ip_list = [ip.strip() for ip in open('./ssd','r')]

def getsize(host):
        now = time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time()))
        print host,now
        command = "df -h"
        sys_cmd = """ssh -n -f -i /usr/home/guosong/.ssh/id_rsa -p 26387 -o StrictHostKeyChecking=no -o ConnectTimeout=2 root@'%s' "%s" """ %(host,command)

        pipe = subprocess.Popen(sys_cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE,shell=True,stdin=subprocess.PIPE)
        stdout,stderr = pipe.communicate()
        #print stdout

        results.append(stdout)

pool = ThreadPool(4)

pool.map(getsize,ip_list)
pool.close()

pool.join()　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　

效率测试：

获取所有端口对应的MySQL版本信息，使用循环和多线程对比

#!/usr/bin/env python2.7
#-*- coding:utf-8 -*-

import urllib2
from multiprocessing.dummy import Pool as ThreadPool
import time

result_list = []
def get_instance(port):
    url = "http://api.xxxr.sina.com.cn/autosystem/get_iplist?token=xxx&username=xxx&port=%d" % port
    reps = urllib2.urlopen(url)

    result = reps.read()

    result_list.append(result)

if __name__ == '__main__':
    port_list = [int(port.strip()) for port in open('./ports','r')]

    '''for port in port_list:
        get_instance(port)'''

    pool = ThreadPool(10)

    pool.map(get_instance,port_list)

    pool.close()

    pool.join()                                                                                                                                              

    print len(result_list)

时间：

[root@typhoeus79 mysql_version]# time ./get_verion.py #多线程方式
real    0m2.298s
user    0m0.353s
sys     0m0.114s

[root@typhoeus79 mysql_version]# time ./get_verion.py #循环方式
real    0m15.341s
user    0m0.356s
sys     0m0.088s

效率有明显提升

map传入多个元素

#!/usr/bin/env python2.7
# -*-coding:utf8 -*-
from multiprocessing.dummy import Pool as ThreadPool

def add(meta):
    a = meta[0]
    b = meta[1]

    print meta
    return a + b 

if __name__ == '__main__':
    pool = ThreadPool(4)

    alist = [1, 2, 3]
    blist = [4, 5, 6]

    pool.map(add, zip(alist, blist))
    pool.close()
    pool.join()

　　

使用zip方式