多进程实例——爬取百度贴吧

上次介绍了多进程并发相关内容，本次以爬取百度贴吧为例，进行实战演示。

爬去的网址：http://tieba.baidu.com/p/3522395718

本次爬去每层楼的发帖人、发帖内容和发帖时间。

闲话不说直接上代码

# -*- coding: utf-8 -*-
import requests
from bs4 import BeautifulSoup
import json
from multiprocessing import Pool
import time

urls = ['http://tieba.baidu.com/p/3522395718?pn={}'.format(i) for i in range(1,20)]

def return_infos(url):
    html = requests.get(url)
    soup = BeautifulSoup(html.content,'lxml')
    items = soup.select('div.p_postlist div.l_post.j_l_post.l_post_bright')
    contents = soup.select('div.p_postlist div.d_post_content.j_d_post_content.clearfix')
    names = soup.select('div.p_postlist li.d_name a')
    for i,j,k in zip(items,contents,names):
        item = json.loads(i.get('data-field'))
        date = item['content']['date']
        content = j.text.strip()
        name = k.text.strip()
        print(name,date,content)

if __name__ == '__main__':
    t0 = time.time()
    for ix in urls:
        return_infos(ix)
    t1 = time.time()

    pool = Pool(4)
    pool.map(return_infos,urls)
    pool.close()  # 关闭进程池，不再接受新的进程
    pool.join()  # 主进程阻塞等待子进程的退出
    t2 = time.time()
    print("正常执行的时间：", (t1 - t0))
    print("并行执行时间：", (t2 - t1))

# -*- coding: utf-8 -*-
import requests
from bs4 import BeautifulSoup
import json
from multiprocessing import Pool
import time

urls = ['http://tieba.baidu.com/p/3522395718?pn={}'.format(i) for i in range(1,20)]

def return_infos(url):
    html = requests.get(url)
    soup = BeautifulSoup(html.content,'lxml')
    items = soup.select('div.p_postlist div.l_post.j_l_post.l_post_bright')
    contents = soup.select('div.p_postlist div.d_post_content.j_d_post_content.clearfix')
    names = soup.select('div.p_postlist li.d_name a')
    for i,j,k in zip(items,contents,names):
        item = json.loads(i.get('data-field'))
        date = item['content']['date']
        content = j.text.strip()
        name = k.text.strip()
        print(name,date,content)

if __name__ == '__main__':
    t0 = time.time()
    for ix in urls:
        return_infos(ix)
    t1 = time.time()

    pool = Pool(4)
    pool.map(return_infos,urls)
    pool.close()  # 关闭进程池，不再接受新的进程
    pool.join()  # 主进程阻塞等待子进程的退出
    t2 = time.time()
    print("正常执行的时间：", (t1 - t0))
    print("并行执行时间：", (t2 - t1))

爬取结果：

正常执行的时间： 16.037917375564575
并行执行时间： 6.655380487442017