分析Ajax请求并抓取今日头条街拍美图图集(进程池、MongoDB、二进制流文件、正则、requests)

流程如下：

#1 配置好MongoDB的依赖库 

#2 模拟搜索街拍的请求信息

#3 通过请求返回的json返回的url地址再次爬取

#4 爬取新的url地址，并爬取相关的图片地址

#5 获取url地址，并将爬取数据写至MongoDB，且通过二进制流下载下来，若文件相同，则通过md5判断

1、相关配置文件如下：

config.py

MONGO_URL = 'localhost:27017'
MONGO_DB = 'toutiao'
MONGO_TABLE = 'toutiao1'

GROUP_START = 0
GROUP_END = 19
KEYWORD='街拍'

2、爬虫代码如下：

import json

from urllib.parse import urlencode
from hashlib import md5  #导入MD5判断

import os
import pymongo
import requests
from requests.exceptions import RequestException
import re
from bs4 import BeautifulSoup as bs
from config import *
from multiprocessing import Pool

#mongodb数据库连接
client=pymongo.MongoClient(MONGO_URL)
db=client[MONGO_DB]

#保存在mongodb上
def save_to_mongo(result):
    if db[MONGO_TABLE].insert(result):
        print('存储到MongoDB成功',result)
        return True
    return False


#01
def get_page_index(offset,keyword):#获取索引页的json
    data={
        'offset': offset,
        'format': 'json',
        'keyword':keyword,
        'autoload':'true',
        'count':20,
        'cur_tab':3
    }
    url='https://www.toutiao.com/search_content/?'+urlencode(data)#将字典对象转为请求参数
    try:
        response=requests.get(url)
        if response.status_code==200:
            return response.text
        return None
    except RequestException:
        print('请求索引页出错')
        return None

#02
def parse_page_index(html):#获取索引页过来的json中的url地址
    data=json.loads(html)#json字符串转换为json对象
    if 'data' in data.keys():
        for item in data.get('data'):
            yield item.get('article_url')

#03
def get_page_detail(url):#获取详情
    try:
        response = requests.get(url)
        if response.status_code == 200:
            return response.text
        return None
    except RequestException:
        print('请求详情页出错',url)
        return None

#04
def parse_page_detail(html,url):
    soup=bs(html,'lxml')
    title=soup.title.string

    image_pattren = re.search(r'gallery:.*?parse("(.*?)"),', html, re.S) #这里一定要加r，表示不转义
    image_pattren = re.sub(r'\','',image_pattren.group(1))
    try:
        data=json.loads(image_pattren)
    except: #有些json需要修复
        image_pattren = "{" + re.search(r'("sub_images":[{.*?}]),"max', image_pattren).group(1) + "}"
        data=json.loads(image_pattren)

    if data and 'sub_images' in data.keys():
        sub_images = data.get('sub_images')
        images = [item.get('url') for item in sub_images]
        for img in images:
            download_img(img)
        return {
            'title':title,
            'url':url,
            'images':images
        }

#05
def download_img(url):#通过地址下载图片的二进制流
    print('正在下载',url)
    try:
        response=requests.get(url)
        if response.status_code==200:
            save_img(response.content)  #保存二进制流为图片
        return None
    except RequestException:
        print('请求图片出错',url)
        return None


#06
def save_img(content):
    file_path='{0}/{1}.{2}'.format('d:/123',md5(content).hexdigest(),'jpg')
    if not os.path.exists(file_path):
        with open(file_path,'wb') as f:
            f.write(content)



def main(offset):
    html=get_page_index(offset,KEYWORD)
    urls=parse_page_index(html)
    for url in urls:
        html=get_page_detail(url)
        result=parse_page_detail(html,url)
        save_to_mongo(result)




if __name__=='__main__':
    group=[x*20 for x in range(GROUP_START,GROUP_END+1)]
    pool = Pool()
    pool.map(main, group)