学习笔记--Python多进程

os.fork

该方法仅支持Linux/Unix

import os

print('Current Process %s start ...' % os.getpid())
pid = os.fork()  #复制主进程以生成一个子进程
if pid < 0:
    print('error in fork ')
elif pid == 0:
    print('I am child process %s and my parent process is %s' % (os.getpid(), os.getppid()))
else:
    print('I %s created a child process %s' % (os.getppid(), os.getpid()))

multiprocessing

进程池 pool

from multiprocessing import Pool
import os, time, random

def run_task(name):
    print('Task %s (pid = %s) is running...' % (name, os.getpid()))
    time.sleep(random.random() * 3)
    print('Task %s end.' % name)

if __name__ == '__main__':
    print('Curret process %s.' % os.getpid())
    p = Pool(processes = 3)  # 创建进程池
    for i in range(5):
        p.apply_async(run_task, args = (i, ))  # 启动进程
    print('Waiting for all subprocesses done...')
    p.close()
    p.join()
    print('All subprocesses done.')

进程间通信

Queue

from multiprocessing import Queue, Process
import os, time, random

def proc_write(q, urls):
    print('Process (%s) is writig...' % os.getpid())
    for url in urls:
        q.put(url) # 往队列中添加
        print('Put %s to queue...' % url)
        time.sleep(random.random())

def proc_read(q):
    print('Process (%s) is writig...' % os.getpid())
    while True:
        url = q.get(True)  # 从队列中取出
        print('Get %s from queue.' % url)

if __name__ == '__main__':
    q = Queue()  # 创建消息队列
    # 创建进程
    proc_writer1 = Process(target = proc_write, args = (q, ['url_1', 'url_2', 'url_3', 'url_4']))
    proc_writer2 = Process(target = proc_write, args = (q, ['url_5', 'url_6', 'url_7', 'url_8']))
    proc_reader = Process(target = proc_read, args = (q, ))
    # 启动进程，往消息队列中添加
    proc_writer1.start()
    proc_writer2.start()
    # 启动读queue进程
    proc_reader.start()
    # 等待proc_writer结束
    proc_writer1.join()
    proc_writer2.join()
    # 由于proc_reader 进程里是死循环，这里强行终止
    proc_reader.terminate()

Pipe

import multiprocessing, random, os, time

def proc_send(pipe, urls):
    for url in urls:
        print('Process %s send: %s ' % (os.getpid(), url))
        pipe.send(url)  # 发送 url
        time.sleep(random.random())

def proc_recv(pipe):
    while True:
        print('Process %s rev: %s' % (os.getpid(), pipe.recv()))
        time.sleep(random.random())

if __name__ == '__main__':
    pipe = multiprocessing.Pipe()
    p1 = multiprocessing.Process(target = proc_send, args = (pipe[0], ['url_' + str(i) for i in range(10)]))
    p2 = multiprocessing.Process(target = proc_recv, args = (pipe[1], ))
    p1.start()
    p2.start()
    p1.join()
    p2.terminate()

代码摘自 《Python爬虫开发与项目实战》-- 范伟辉