scrapy+pyppeteer指定搜索动态爬取头条

一、介绍

由于头条现在采取了动态js渲染的反爬措施，还有其他各种js加密反爬，使用简单的requests非常困难

Puppeteer 是 Google 基于 Node.js 开发的一个工具，有了它我们可以通过 JavaScript 来控制 Chrome 浏览器的一些操作，当然也可以用作网络爬虫上，其 API 极其完善，功能非常强大。 而 Pyppeteer 又是什么呢？它实际上是 Puppeteer 的 Python 版本的实现，但他不是 Google 开发的，是一位来自于日本的工程师依据 Puppeteer 的一些功能开发出来的非官方版本。

详细使用方法，官方文档: https://miyakogi.github.io/pyppeteer/reference.html

二、简单的使用

import asyncio
from pyppeteer import launch

async def main():
    # browser = await launch()
    browser = await launch(headless=True)
    page = await browser.newPage()
    await page.setViewport(viewport={'width': 1366, 'height': 768})
    # await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3494.0 Safari/537.36')

    await page.goto('https://www.toutiao.com')
    # 是否启用js
    await page.setJavaScriptEnabled(enabled=True)
    await page.evaluate(
        '''() =>{ Object.defineProperties(navigator,{ webdriver:{ get: () => false } }) }''')  # 以下为插入中间js，将淘宝会为了检测浏览器而调用的js修改其结果。
    await page.evaluate('''() =>{ window.navigator.chrome = { runtime: {},  }; }''')
    await page.evaluate(
        '''() =>{ Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] }); }''')
    await page.evaluate(
        '''() =>{ Object.defineProperty(navigator, 'plugins', { get: () => [1, 2, 3, 4, 5,6], }); }''')


    await page.goto('https://www.toutiao.com/search/?keyword=%E5%B0%8F%E7%B1%B310')

    # 打印cookie页面
    print(await page.cookies())

    await asyncio.sleep(5)

    # # 打印页面文本
    print(await page.content())
    #
    # # 打印当前首页的标题
    print(await page.title())
    with open('toutiao.html', 'w', encoding='utf-8') as f:
        f.write(await page.content())

    await browser.close()

loop = asyncio.get_event_loop()
task = asyncio.ensure_future(main())
loop.run_until_complete(task)

三、多次请求的使用

import asyncio
from pyppeteer import launch


async def main(url):
    browser = await launch(headless=True)
    page = await browser.newPage()
    await page.setViewport(viewport={'width': 1366, 'height': 768})
    await page.setUserAgent(
        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3494.0 Safari/537.36')

    # 是否启用js
    await page.setJavaScriptEnabled(enabled=True)

    await page.evaluate(
        '''() =>{ Object.defineProperties(navigator,{ webdriver:{ get: () => false } }) }''')  # 以下为插入中间js，将淘宝会为了检测浏览器而调用的js修改其结果。
    await page.evaluate('''() =>{ window.navigator.chrome = { runtime: {},  }; }''')
    await page.evaluate(
        '''() =>{ Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] }); }''')
    await page.evaluate(
        '''() =>{ Object.defineProperty(navigator, 'plugins', { get: () => [1, 2, 3, 4, 5,6], }); }''')

    await page.goto(url,options={'timeout': 5000})

    # await asyncio.sleep(5)
    # 打印页面文本
    return await page.content()

tlist = ["https://www.toutiao.com/a6794863795366789636/",
             "https://www.toutiao.com/a6791790405059871236/",
             "https://www.toutiao.com/a6792756350095983104/",
             "https://www.toutiao.com/a6792852490845946376/",
             "https://www.toutiao.com/a6795883286729064964/",
         ]

task = [main(url) for url in tlist]

loop = asyncio.get_event_loop()
results = loop.run_until_complete(asyncio.gather(*task))
for res in results:
    print(res)

四、scarpy+pyppeteer

github地址：https://github.com/fengfumin/Toutiao_pyppeteer

主要功能：

1、指定搜素单词

from scrapy.cmdline import execute
execute(['scrapy', 'crawl', 'toutiao','-a','keyword=小米10'])

2、爬取页面内存保存csv

3、生成词云图片

 4、主要代码scrapy中间件

from scrapy import signals
import pyppeteer
import asyncio
import os
from scrapy.http import HtmlResponse

pyppeteer.DEBUG = False


class ToutiaoPyppeteerDownloaderMiddleware(object):
    # Not all methods need to be defined. If a method is not defined,
    # scrapy acts as if the downloader middleware does not modify the
    # passed objects.
    def __init__(self):
        print("Init downloaderMiddleware use pypputeer.")
        os.environ['PYPPETEER_CHROMIUM_REVISION'] = '588429'
        # pyppeteer.DEBUG = False
        print(os.environ.get('PYPPETEER_CHROMIUM_REVISION'))
        loop = asyncio.get_event_loop()
        task = asyncio.ensure_future(self.getbrowser())
        loop.run_until_complete(task)

        # self.browser = task.result()
        print(self.browser)
        print(self.page)
        # self.page = await browser.newPage()

    async def getbrowser(self):
        self.browser = await pyppeteer.launch(headless=True)
        self.page = await self.browser.newPage()
        await self.page.setViewport(viewport={'width': 1366, 'height': 768})
        await self.page.setUserAgent(
            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3494.0 Safari/537.36')
        await self.page.setJavaScriptEnabled(enabled=True)
        await self.page.evaluate(
            '''() =>{ Object.defineProperties(navigator,{ webdriver:{ get: () => false } }) }''')  # 以下为插入中间js，将淘宝会为了检测浏览器而调用的js修改其结果。
        await self.page.evaluate('''() =>{ window.navigator.chrome = { runtime: {},  }; }''')
        await self.page.evaluate(
            '''() =>{ Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] }); }''')
        await self.page.evaluate(
            '''() =>{ Object.defineProperty(navigator, 'plugins', { get: () => [1, 2, 3, 4, 5,6], }); }''')

        return self.page


    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    def process_request(self, request, spider):
        # Called for each request that goes through the downloader
        # middleware.

        # Must either:
        # - return None: continue processing this request
        # - or return a Response object
        # - or return a Request object
        # - or raise IgnoreRequest: process_exception() methods of
        #   installed downloader middleware will be called
        loop = asyncio.get_event_loop()
        task = asyncio.ensure_future(self.usePypuppeteer(request))
        loop.run_until_complete(task)
        # return task.result()
        return HtmlResponse(url=request.url, body=task.result(), encoding="utf-8", request=request)

    async def usePypuppeteer(self, request):
        print(request.url)
        # self.page = await self.getbrowser()
        await self.page.goto(request.url)
        await asyncio.sleep(3)
        #鼠标滚动到底
        await self.page.evaluate('window.scrollBy(0, document.body.scrollHeight)')
        await asyncio.sleep(3)
        content = await self.page.content()
        return content

    def process_response(self, request, response, spider):
        # Called with the response returned from the downloader.

        # Must either;
        # - return a Response object
        # - return a Request object
        # - or raise IgnoreRequest
        return response

    def process_exception(self, request, exception, spider):
        # Called when a download handler or a process_request()
        # (from other downloader middleware) raises an exception.

        # Must either:
        # - return None: continue processing this exception
        # - return a Response object: stops process_exception() chain
        # - return a Request object: stops process_exception() chain
        pass

    def spider_opened(self, spider):
        spider.logger.info('Spider opened: %s' % spider.name)

5、主要代码请求与处理

import csv
import time

from scrapy import Spider, Request
from bs4 import BeautifulSoup
from Toutiao_pyppeteer.cloud_word import *


class TaobaoSpider(Spider):
    name = 'toutiao'
    allowed_domains = ['www.toutiao.com']
    start_url = ['https://www.toutiao.com/','https://www.toutiao.com/search/?keyword={keyword}']

    def __init__(self, keyword, *args, **kwargs):
        super(TaobaoSpider, self).__init__(*args, **kwargs)
        self.keyword = keyword


    def open_csv(self):
        self.csvfile=open('train/data.csv', 'w', newline='', encoding='utf-8')
        fieldnames = ['title', 'comment']
        self.dict_writer = csv.DictWriter(self.csvfile, delimiter=',', fieldnames=fieldnames)
        self.dict_writer.writeheader()

    def close_csv(self):
        self.csvfile.close()

    def start_requests(self):
        for url in self.start_url:
            if 'search' in url:
                r_url = url.format(keyword=self.keyword)
            else:
                r_url=url
            yield Request(r_url, callback=self.parse_list)

    def parse_list(self, response):
        if "小米10" in response.text:
            soup = BeautifulSoup(response.text, 'lxml')  # 具有容错功能
            res = soup.prettify()  # 处理好缩进，结构化显示
            div_list=soup.find_all('div', class_='articleCard')
            # print(res)
            print(div_list)
            print(len(div_list))
            self.open_csv()
            for div in div_list:
                title=div.find('span',class_='J_title')
                self.dict_writer.writerow({"title":title.text})
                con=div.find('div',class_='y-left')
                self.dict_writer.writerow({"comment": con.text})

            print("关闭csv")
            self.close_csv()
            print("开始分词")
            text_segment()
            print("开始制图")
            chinese_jieba()