Scrapy操作浏览器获取网易新闻数据

爬虫代码：

import scrapy
from selenium import  webdriver

class WangyiSpider(scrapy.Spider):
    name = 'wangyi'
    # allowed_domains = ['www.xxx.com']
    start_urls = ['https://news.163.com/']

    def __init__(self):
        self.bro = webdriver.Chrome(r'D:爬虫相关资料驱动程序chromedriver_win32chromedriver.exe')

    def parse(self, response):
        #解析国内国际军事航空对应的url
        li_list=response.xpath('//div[@class="ns_area list"]/ul/li')
        #国内国际军事航空对应的索引
        index_url=[3,4,6,7]
        #存储四个板块对应的li标签
        news_list=[]
        for i in index_url:
            news_list.append(li_list[i])
        #解析获取板块的url
        for li in news_list:
            url=li.xpath('./a/@href').extract_first()
            yield  scrapy.Request(url=url,callback=self.parse_news)

    def parse_news(self,response):
        print('xxx:',response.xpath('/html/body/div[1]/div[3]/div[4]/div[1]/div/div/ul/li/div/div[3]/div[1]/h3/a/text()').extract_first())
    def closed(self,spider):
        self.bro.quit()

middewares中间件代码：

from scrapy import signals
from  scrapy.http import HtmlResponse



class WangyiproDownloaderMiddleware(object):
    def process_response(self, request, response, spider):
        if request.url in ['http://news.163.com/air/','http://war.163.com/','http://news.163.com/world/','http://news.163.com/domestic/']:
            spider.bro.get(request.url)
            page_text=spider.bro.page_source#获取数据源
            return HtmlResponse(url=spider.bro.current_url,body=page_text,encoding='utf-8',request=request)
        return response