multiprocessing.pool 使用

1. map ,map_async

# f 参数是两个，multiprocessing.pool.map框架只能传一个的时候

from multiprocessing import Pool
import time

# 1 这个方法不行，但是装饰器思路好
# def my_function_helper(func):
#     def inner(tup):
#         print('start wrapper is {}'.format(tup))
#         r = func(*tup)
#         print('end wrapper')
#         return r
#     return inner
#
#
# @my_function_helper
# def f(x,y):
#     print (x*x)
#     print(y)


# 2 这个方法可行，不用改变原函数
def f(x,y):
    print (x*x,y)


def my_function_helper(tup):
    return f(*tup)


if __name__ == '__main__':
    pool = Pool(processes=4)

    # 一个参数的情况
    # pool.map(f, [i for i in range(10)])
    # r = pool.map_async(f, [i for i in range(10)])

    # 两个参数的情况
    pool.map(my_function_helper, [(i,2) for i in range(10)])
    r = pool.map_async(my_function_helper, [(i,2) for i in range(10)])
    # DO STUFF
    print ('HERE')
    print ('MORE')
    r.wait()
    print ('DONE')

2. starmap (类似map是同步的，看例子)

from multiprocessing import Pool, cpu_count
import time


build_links = [1,2,3,4,5,6,7,8]
auth = 'auth'


def test(url, auth):
    time.sleep(2)
    with open('1.txt',mode='w') as f:
        f.write('test')
    print(url, auth)

def test2(url, auth):
    time.sleep(6)
    with open('2.txt',mode='w') as f:
        f.write('test2')
    print(url, auth)

if __name__ == '__main__':  # 必须要加，不然出错,以避免递归创建子流程。
    with Pool(processes=int(cpu_count() - 1) or 1) as pool:
        pool.starmap(test, [(link, auth) for link in build_links])
        print('要等上面的函数执行完成！')
        pool.starmap(test2, [(link, auth) for link in build_links])

3. 多参数异步的实现

from multiprocessing import Pool, cpu_count
import time

build_links = [1,2,3,4,5,6,7,8]
auth = 'auth'


def test(url, auth):
    time.sleep(2)
    with open('1.txt',mode='w') as f:
        f.write('test')
    print(url, auth)

def test2(url, auth):
    time.sleep(6)
    with open('2.txt',mode='w') as f:
        f.write('test2')
    print(url, auth)


def my_test_helper(tup):
    return test(*tup)


def my_test2_helper(tup):
    return test2(*tup)


if __name__ == '__main__':  # 必须要加，不然出错,以避免递归创建子流程。
    with Pool(processes=int(cpu_count() - 1) or 1) as pool:
        print('1')
        r = pool.map_async(my_test_helper, [(link, auth) for link in build_links])
        r2 = pool.map_async(my_test2_helper, [(link, auth) for link in build_links])
        print('2')
        r.wait()
        print('MORE')
        r2.wait()
        print('DONE')