用来爬取新浪微博评论数据

# -*- coding:utf-8 -*-
import requests
import json
import re
import os
import gevent
import time
import random
from multiprocessing.dummy import Pool as ThreadPool
from bs4 import BeautifulSoup
class CommentCrawl(object):
    '''
    用来爬取新浪微博评论数据
    '''
    headers = {
        'User-Agent': '',
        'Cookie': ''}
    ALPHABET = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
    all_comment = []
    def __init__(self,urlll,file_name):
        self.urlll = urlll
        self.file_name=file_name
    def base62_decode(self,string, alphabet=ALPHABET):
        base = len(alphabet)
        strlen = len(string)
        num = 0
        idx = 0
        for char in string:
            power = (strlen - (idx + 1))
            num += alphabet.index(char) * (base ** power)
            idx += 1
        return num

    def parser_url(self):
        code = self.urlll.split('?')[0].split('/')[-1]
        code1 = code[0]
        code2 = code[1:5]
        code3 = code[5:]
        id1 = self.base62_decode(code1)
        id2 = self.base62_decode(code2)
        id3 = self.base62_decode(code3)
        numList = [id1, id2, id3]
        plus = ''.join(map(str, numList))
        comment_url = 'http://weibo.com/aj/v6/comment/big?ajwvr=6&id='+ plus +'&root_comment_max_id_type=0&page={}'
        return comment_url
    def get_url_page(self):
        r = requests.get(self.parser_url().format(1),headers=self.headers)
        data = json.loads(r.text)
        total_page = data['data']['page']['totalpage']
        return total_page

    def all_urls(self):
        all_urls = [self.parser_url().format(i + 1) for i in range(self.get_url_page())]
        return all_urls

    def comment_parser(self,html):
        soup = BeautifulSoup(html, 'html.parser')
        data = soup.select('.WB_text')
        comment = [i.text.split('：')[-1] for i in data]
        return comment
    def finnal_text(self,url):
        finnal_all_comment=''.join(self.all_comment)
        r1 = requests.get(url,headers=self.headers)
        time.sleep(random.randint(1,5))
        data1 = json.loads(r1.text)
        html =data1['data']['html']
        finnal_data = self.comment_parser(html)
        self.all_comment+=finnal_data
        print(len(self.all_comment))
        return finnal_all_comment
    def save_file(self,url):
        path = os.getcwd()
        filename = self.file_name + '.txt'
        file = path + '/' + filename
        f = open(file, 'a+', encoding='utf-8')
        f.write(self.finnal_text(url))

if __name__ == "__main__":
    aa = CommentCrawl('http://weibo.com/2202387347/EFdPHe50Z?from=page_1006062202387347_profile&wvr=6&mod=weibotime','小米6发布会')
    all_link = aa.all_urls()
    pool=ThreadPool(4)
    results = pool.map(aa.save_file,all_link)
    pool.close()
    pool.join()