爬虫

爬虫 - 斗鱼房间信息

'''爬取斗鱼直播所有的房间信息,
https://www.douyu.com/gapi/rkc/directory/0_0/1 还可直接获取json数据
'''
import time

from selenium import webdriver
from fake_useragent import UserAgent
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC

class DouYuRoom(object):
    def __init__(self):
        ua = UserAgent().random
        self.url = 'https://www.douyu.com/directory/all'
        # self.driver = webdriver.PhantomJS(r'F:/tools/phantomjs-2.1.1-windows/bin/phantomjs.exe')
        self.driver = webdriver.Chrome()
        self.content_list = []
    def get_content_list(self):

        li_list = self.driver.find_elements_by_xpath('//*[@id="listAll"]/section[2]/div[2]/ul/li')
        content_list = []
        for li in li_list:
            content_dic = {}
            game = li.find_element_by_xpath('.//span[@class="DyListCover-zone"]').text
            room_url = li.find_element_by_xpath('.//a').get_attribute('href')
		   #.....
            content_dic['game'] = game
            content_dic['room_url'] = room_url
            content_list.append(content_dic)
        next_url = self.driver.find_elements_by_xpath('//*[@id="listAll"]/section[2]/div[2]/div/ul/li[9]/span')
        next_url =next_url[0] if len(next_url)>0 else None
        print('next_url',next_url)
        return next_url,content_list

    def save_content(self,content):
        pass

    def run(self):
        # 获取数据
        self.driver.get(self.url)
        # 等待加载完毕，可以使用显示等待，隐式等待
        time.sleep(10)
        # 提取数据
        next_url,content_dic = self.get_content_list()
        print(content_dic)
        # 每次的数据都先保存下来了
        self.save_content(content_dic)
        # 开始下一页的提取
        while next_url:
            next_url.click()
            time.sleep(5)
            next_url, content_dic = self.get_content_list()
            print(content_dic)
            self.save_content(content_dic)
        # 关闭浏览器
        self.driver.close()


if __name__ == '__main__':
    dy = DouYuRoom()
    dy.run()