python 简书用户爬虫

python 简书用户爬虫 广度优先

# python
# -*- coding: utf-8 -*-
"""
__title__ = ''
__author__ = 'wlc'
__mtime__ = '2017/10/15'
"""
import re
import time
import math
import csv
import requests
from bs4 import BeautifulSoup
from collections import deque
import sys
#python 默认递归限制为900
sys.setrecursionlimit(10000)

#建立一个csv文件保存信息
path = 'dataCollection/userInfo.csv'
csvFile = open(path, 'a+', newline='', encoding='utf-8')
writer = csv.writer(csvFile)
writer.writerow(('id','name','following','follower','article','word','like'))

#全局变量用来存储userid 和关注的人数
idContainer = set()
#用来放置用户的链接使用双向队列
linkDeque  = deque()

class jianshu(object):
    def __init__(self):
        #定制url模板
        self.url = 'http://www.jianshu.com/users/{userId}/following?page={page}'
        #用户id与name的匹配规则
        self.idPattern = re.compile('<a class="name" href="/u/(.*?)">(.*?)</a>')
        #用户的关注 粉丝 文章 文集 的匹配规则
        self.metalPattern = re.compile('<span>关注 (d+)</span><span>粉丝 (d+)</span><span>文章 (d+)</span>')
        self.meta = re.compile('写了 (d+) 字，获得了 (d+) 个喜欢')
        #伪装成浏览器
        self.header = {"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36"}

    def createRequest(self, userId, page):
        url = self.url.format(userId = userId, page = page)
        requ = requests.get(url, headers = self.header).text
        return requ

    def pageResponse(self, requ):
        bsOBJ = BeautifulSoup(requ, 'lxml')
        userContainer = bsOBJ.find_all('ul',{'class':'user-list'})[0]
        userContent = userContainer.contents
        userContent = [str(user) for user in userContent if user != '
']
        #关注用户列表
        return userContent

    def parserUserInfo(self, user):
        id, name = re.findall(self.idPattern, user)[0]
        followingNum, followerNum, articleNum = re.findall(self.metalPattern, user)[0]
        try:
            wordNum, likeNum = re.findall(self.meta, user)[0]
        except:
            wordNum, likeNum = 0, 0
        content = (id, name, followingNum, followerNum, articleNum, wordNum, likeNum)
        writer.writerow(content)
        return  content

    def getUserList(self, userId, following):
        idContainer.add((userId, following))
        num = int(following) / 10
        page = math.ceil(num)
        for pg in range(1, page + 1, 1):
            requ = self.createRequest(userId, pg)
            userList = self.pageResponse(requ)
            for user in userList:
                content = self.parserUserInfo(user)
                linkDeque.append((content[0], content[2]))
            time.sleep(1)
        for deq in linkDeque:
            if deq not in idContainer:
                self.getUserList(deq[0],deq[1])
                print("what")
jianshu = jianshu().getUserList('1562c7f16a04',45)