爬虫之知乎用户信息爬取

这个爬虫程序有别于之前写的两个，这个是自己写的，使用的是python库requests、redis、lxml。

一共有三个文件分别是config.ini用户名和cookie配置文件，zhihusp.py爬取用户名，get-info.py爬取用户其他信息。

下面分别将三个文件贴出来，两个python文件注释比较详细，应该都看得懂。

config.ini

[info]
phone_num = 15*********
password = ************

[cookies]
q_c1 = 5fd5e96aa1cc40f587e2fcaa621030ee|1448986627000|1448986627000
cap_id = Zjk3N2I3MjU1ZmIyNGJkNWJIDOxYmE3ZDEzN2QyOGE=|1449289675|
612bbfbnjd2e3bca76d397a2c67c921fe7c852b
_za = b7e8ab32-03b3-473b-87e6-68fe9f9e7933
__utmt = 1
__utma = 51854390.1168696635.1449128833.1449239113.1449289659.5
__utmb = 51854390.6.10.1449289659
__utmc = 51854390
__utmz = 51854390.1449223233.4.2.utmcsr=zhihu.coccn=(referral)|
utmcmd=referral|utmcct=/people/excited-vczh/followers
__utmv = 51854390.100-2|2=re=1^3=entry_date=20151202=1
z_c0 = QUJDTXpzbTNGd2tYQUFBdffabXowaVZZdHBZbnJIS3FhYjZBQnRTWllWQlZ1T
1kyc1dnPT0=|1449289708|7020f5e7c6c95b043e48c02afffb3a9c40035a77
unlock_ticket = QUJDTXpzbTNGd2tYQUFBQVlRSlZUZlJ1WWxaUDlzRGpZTVocGdn
Ul8xZkVNbDNBPT0=|1554289708|d906b57006b0cd84c58c4f6d6e1eb16e17e64

zhihusp.py 主要用户从关注着列表抓取关注者id

# -*- coding: utf-8 -*-
'''
网络爬虫之爬取知乎用户信息
'''
import requests, json, re, redis, sqlite3
import ConfigParser
from lxml import etree
import sys
reload(sys)
sys.setdefaultencoding("utf-8")

class ZhihuSpider(object):
    """docstring for ZhihuSpider"""
    r = redis.Redis(host='127.0.0.1',port=6379,db=1)
    cf = ConfigParser.ConfigParser()
    cf.read('config.ini')
    cookies = cf.items('cookies')
    cookies = dict(cookies)
    session = requests.session()
    conn = sqlite3.connect('zhihuuser.db')
    conn.text_factory = str
    cur = conn.cursor()

    # 创建链接，如果使用用户名、密码登录不上，则改用cookie登录
    def create_session(self):
        
        from pprint import pprint
        pprint(self.cookies)
        phone_num = self.cf.get('info', 'phone_num')
        password = self.cf.get('info', 'password')
        login_data = {'phone_num': phone_num, 'password': password}
        header = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) 
            AppleWebKit/537.36 
            (KHTML, like Gecko) Chrome/43.0.2357.124 Safari/537.36',
            'Host': 'www.zhihu.com',
            'Referer': 'http://www.zhihu.com/'
        }
        r = self.session.post('http://www.zhihu.com/login/phone_num',
         data=login_data,
         headers=header)
        if r.json()['r'] == 1:
            print 'Login Failed, reason is:',
            for m in r.json()['data']:
                print r.json()['data'][m]
            print 'So we use cookies to login in...'
            has_cookies = False
            for key in self.cookies:
                if key != '__name__' and self.cookies[key] != '':
                    has_cookies = True
                    break
            if has_cookies is False:
                raise ValueError('请填写config.ini文件中的cookies项.')
            else:
                r=self.session.get('http://www.zhihu.com/login/phone_num',
                 cookies=self.cookies) # 实现cookie登陆

        with open('login.html', 'w') as fp:
            fp.write(r.content)

    # 请求用户关注者或关注了谁的页面
    def follow(self, userid):
        print "NOW Follow:",userid
        self.r.set(userid, False)
        follower_url = "http://www.zhihu.com/people/"+userid+"/followers"
        follower, followee, user_urls = self.getinfo(userid)
        # print user_urls
        for u_url in user_urls:
            userid = u_url.split('/')[-1]
            # print "FFFFFFFLLLLLLLL@*******",userid
            if self.not_in(userid):
                self.r.set(userid, True)
        # print type(follower),follower
        if follower > 20:
            self.doprofiles(follower,follower_url)

        #提取关注的人的第一页的userid
        followee_url = "http://www.zhihu.com/people/"+userid+"/followees"
        response=self.session.get(followee_url,cookies=self.cookies)
            .content
        page = etree.HTML(response)
        user_urls = page.xpath('//h2/a[class="zg-link"]/@href')
        
        for u_url in user_urls:
            userid = u_url.split('/')[-1]
            # print "WWWWWWWW*****",userid
            if self.not_in(userid):
                self.r.set(userid, True)
        if followee > 20:
            self.doprofiles(followee,followee_url)

    # 动态获取“更多”里面的内容
    def doprofiles(self,attention,url):
        thisheader = {
            'Host': 'www.zhihu.com',
            'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64)
             AppleWebKit/537.36 (KHTML,like Gecko) Chrome/43.0.2357.124
              Safari/537.36',
            'Accept': '*/*',
            'Accept-Language': 'zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3',
            'Accept-Encoding': 'gzip, deflate',
            'Content-Type':'application/x-www-form-urlencoded;
                               charset=UTF-8',
            'X-Requested-With': 'XMLHttpRequest',
            'Pragma': 'no-cache',
            'Cache-Control': 'no-cache',
            'Referer': url,
            'Content-Length': 171,
            'Cookie': '填自己的'
        }
        hash_id = '填自己的'
        xsrf = '填自己的'
        # 计算页数，获取更多里面的关注者信息
        pages = attention/20 + 1
        # if pages > 600:
        #     pages = 600
        for x in xrange(1,pages):
            offset = x * 20
            params = json.dumps({"offset":offset,
                "order_by":"created",
                "hash_id":hash_id})
            payload = {"method":"next","params":params,"_xsrf":xsrf}
            content = self.session.post("http://www.zhihu.com/node/
            ProfileFollowersListV2",headers = thisheader
            ,data = payload).content
            load = json.loads(content)
            # print type(load)
            lists = load['msg']
            for item in lists:
                try:
                    userpeople = re.search(r'people/[w+d+-]+',item)
                    # print userpeople
                    if userpeople is not None:
                        people = userpeople.group()
                        userid = people.split('/')[-1]
                        print "PPPPPPPPPPPPPP-------",userid
                        if self.not_in(userid):
                            self.r.set(userid, True)
                except AttributeError:
                    print "ERROR"
                # self.num += 1
        self.gofollow()


    # 继续Follow
    def gofollow(self):
        for key in self.r.keys():
            if self.r.get(key) == 'True':
                self.follow(key)

    # 检查用户名是否在redis里已经存在
    def not_in(self, userid):
        if self.r.exists(userid):
            return False
        else:
            return True

    def getinfo(self,userid):
        follower_url = "http://www.zhihu.com/people/"+userid+"/followers"
        response = self.session.get(follower_url, cookies=self.cookies)
            .content
        page = etree.HTML(response)
        user_urls = page.xpath('//h2/a[@class="zg-link"]/@href')
        # 获取姓名、城市、工作、性别、教育等信息，并存入数据库
        followee = int(page.xpath('//div[@class="zm-profile-side-
        following zg-clear"]/a[1]/strong/text()')[0])
        follower = int(page.xpath('//div[@class="zm-profile-side-
        following zg-clear"]/a[2]/strong/text()')[0])
        return follower, followee, user_urls

if __name__ == '__main__':
    zhihu = ZhihuSpider()
    # 创建表
    zhihu.cur.execute('''create table if not exists userstb
        (userid text primary key,
        username text, gender text, followee integer,
        follower integer, location text,
        business text, employment text, 
        position text, education text, college text, 
        question_num integer, answer_num text)''')
    zhihu.conn.commit()
    zhihu.create_session()

    # 几个知乎大V
    first_users = ['excited-vczh', 'warfalcon','gejinyuban']
    for user in first_users:
        if zhihu.r.exists(user):
            continue
        else:
            zhihu.follow(user)
    # 从redis里面查找没有被follow的用户id
    for key in zhihu.r.keys():
        if zhihu.r.exists(key):
            if zhihu.r.get(key)=='True':
                zhihu.follow(key)

get-info.py 主要访问每个id的主页，提取信息

# -*- coding: utf-8 -*-
'''
网络爬虫之爬取知乎用户信息
'''
import requests, json, re, redis, sqlite3
import ConfigParser
from lxml import etree
from time import ctime
import sys
reload(sys)
sys.setdefaultencoding("utf-8")

class GetInfo(object):
    """docstring for GetInfo"""

    r1 = redis.Redis(host='127.0.0.1',port=6379,db=1)
    r2 = redis.Redis(host='127.0.0.1',port=6379,db=2)
    cf = ConfigParser.ConfigParser()
    cf.read('config.ini')
    cookies = cf.items('cookies')
    cookies = dict(cookies)
    session = requests.session()
    conn = sqlite3.connect('zhihuuser.db')
    cur = conn.cursor()
    itemlist = []
    useridlist = []
    flag = 0

    # 请求用户主页，获取信息，并存入数据库
    def getinfo(self, userid):
        url = "http://www.zhihu.com/people/"+userid
        print "GET:%s---%s" %(userid,ctime())

        # 异常处理，必要！！
        try:
            response = self.session.get(url,cookies=self.cookies).content
            page = etree.HTML(response)
            username = page.xpath('//div[@class="title-section ellipsis"]
            /span[@class="name"]/text()')[0]
            location = page.xpath('//div[@data-name="location"]/span
            /span[@class="location item"]/@title')
            business = page.xpath('//div[@data-name="location"]/span
            /span[@class="business item"]/@title')
            gendertit = page.xpath('//div[@data-name="location"]/span
            /span[@class="item gender"]/i/@class')
            # 没办法直接取出性别，曲线救国
            if len(gendertit)==0:
                gender = 'notsure'
            elif re.search(r'female', gendertit[0]):
                gender = u'女'
            else:
                gender = u'男'
            employment = page.xpath('//div[@data-name="employment"]
                /span/span[@class="employment item"]/@title')
            position = page.xpath('//div[@data-name="employment"]
                /span/span[@class="position item"]/@title')
            education = page.xpath('//div[@data-name="education"]
                /span/span[@class="education item"]/@title')
            college = page.xpath('//div[@data-name="education"]
                /span/span[@class="education-extra item"]/@title')
            followee = int(page.xpath('//div[@class="zm-profile-side-
                following zg-clear"]/a[1]/strong/text()')[0])
            follower = int(page.xpath('//div[@class="zm-profile-side-
                following zg-clear"]/a[2]/strong/text()')[0])
            question_num = int(page.xpath('//div[@class="profile-navba
            r clearfix"]/a[2]/span/text()')[0])
            answer_num =int(page.xpath('//div[@class="profile-navbar
             clearfix"]/a[3]/span/text()')[0])

            # 有些字段用户没有填写，所以需要判断是否为空
            if len(location) == 0:
                location = None
            else:
                location = location[0]
            if len(business) == 0:
                business = None
            else:
                business = business[0]
            if len(employment) == 0:
                employment = None
            else:
                employment = employment[0]
            if len(position) == 0:
                position = None
            else:
                position = position[0]
            if len(education) == 0:
                education = None
            else:
                education = education[0]
            if len(college) == 0:
                college = None
            else:
                college = college[0]

            # 存入数据库并提交
            item = (userid,username,gender,followee,follower,
            location,business,employment,position,education,
            college,question_num,answer_num)
            print userid,username
            has_in = self.cur.execute("insert into userstb 
                values(?,?,?,?,?,?,?,?,?,?,?,?,?)",item)
            self.conn.commit()
            if has_in:
                print u"存入成功"
                self.r2.set(userid,True)
            else:
                print u"存入失败"
        except requests.exceptions.RequestException:
            print u'连接异常'
            self.main()
        except Exception:
            self.r2.set(userid,True)
            self.main()

    # 主循环，从redis里面取出没有查询过的
    def main(self):
        while True:
            for key in self.r1.keys():
                if self.r2.exists(key):
                    continue
                else:
                    self.getinfo(key)

if __name__ == '__main__':
    begin = GetInfo()
    begin.main()

GG