scrapy实战5 POST方法抓取ajax动态页面(以慕课网APP为例子)：

在手机端打开慕课网，fiddler查看如图注意圈起来的位置

经过分析只有画线的page在变化

上代码：

items.py

import scrapy


class ImoocItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    #id
    id=scrapy.Field()
    #标题
    title=scrapy.Field()
    #类别
    tag=scrapy.Field()
    #内容
    content=scrapy.Field()
    #爬取时间
    crawl_time=scrapy.Field()

spiders/IMooc.py

# -*- coding: utf-8 -*-
import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
import json
from imooc.items import ImoocItem
from datetime import datetime

from scrapy.http import FormRequest
class ImoocSpider(scrapy.Spider):
    name = 'IMooc'
    allowed_domains = ['www.imooc.com']
    headers={
        "User - Agent":"mukewang/5.1.4 (Android 4.4.2; Huawei H60-L02 Build/HDH60-L02),Network WIFI",
        "APP-INFO":"mukewang/5.1.4 (Android 4.4.2; Huawei H60-L02 Build/HDH60-L02),Network WIFI",
        "Host":"www.imooc.com",
        "Connection":"Keep-Alive",
        "Content-Type":"application/x-www-form-urlencoded"
    }
    def start_requests(self):
        url="http://www.imooc.com/api3/articlelist"
        requests = []
        for i in range(1,60):
            formdata={
                "typeid":"18",
                 "page":str(i),
                }
            request = FormRequest(url, callback=self.parse_item, formdata=formdata,headers=self.headers)
            requests.append(request)
        return requests
    def parse_item(self, response):
        datas=json.loads(response.text)["data"]
        for data in datas:
            item=ImoocItem()
            item["id"]=data["id"]
            item["title"]=data["title"]
            item["tag"]=data["type"]
            item["content"]=data["desc"]
            item["crawl_time"]=datetime.now()
            yield item

pipelines.py

import pymysql
class ImoocPipeline(object):
    def process_item(self, item, spider):
        con = pymysql.connect(host="127.0.0.1", user="youusername", passwd="youpassword", db="imooc", charset="utf8")
        cur = con.cursor()
        sql = ("insert into imooc_shouji(id,title,tag,content,crawl_time)"
               "VALUES(%s,%s,%s,%s,%s)")
        lis = (item["id"],item["title"],item["tag"],item["content"],item["crawl_time"])
        cur.execute(sql, lis)
        con.commit()
        cur.close()
        con.close()
        return item

settings.py

BOT_NAME = 'imooc'

SPIDER_MODULES = ['imooc.spiders']
NEWSPIDER_MODULE = 'imooc.spiders'
ROBOTSTXT_OBEY = False
DOWNLOAD_DELAY = 5
ITEM_PIPELINES = {
   'imooc.pipelines.ImoocPipeline': 300,
}

只爬取python相关的手记如下图：