zoukankan      html  css  js  c++  java
  • Python爬微博(ajax+mongo)

    import requests
    from urllib.parse import urlencode
    from pyquery import PyQuery as pq
    from pymongo import MongoClient
    
    base_url = 'https://m.weibo.cn/api/container/getIndex?'
    headers = {
        'Host': 'm.weibo.cn',
        'Referer': 'https://m.weibo.cn/u/2145291155',
        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36',
        'X-Requested-With': 'XMLHttpRequest',
    }
    client = MongoClient()
    db = client['weibo']
    collection = db['weibo']
    max_page = 14
    
    
    def get_page(page):
        params = {
            'type': 'uid',
            'value': '2145291155',
            'containerid': '1076032145291155',
            'page': page
        }
        url = base_url + urlencode(params)
        try:
            response = requests.get(url, headers=headers)
            if response.status_code == 200:
                return response.json()
        except requests.ConnectionError as e:
            print('Error', e.args)
    
    
    def parse_page(json):
        if json:
            items = json.get('cards')
            for item in items:
                item = item.get('mblog')
                weibo = {}  #字典
                weibo['id'] = item.get('id')
                weibo['text'] = pq(item.get('text')).text()
                weibo['attitudes'] = item.get('attitudes_count')
                weibo['comments'] = item.get('comments_count')
                weibo['reposts'] = item.get('reposts_count')
                yield weibo
    
    
    def save_to_mongo(result):
        if collection.insert(result):
            print('Saved to Mongo')
    
    
    if __name__ == '__main__':
        for page in range(1, max_page + 1):
            json = get_page(page)
            results = parse_page(json)
            for result in results:
                print('
    '.result)
                save_to_mongo(result)
    
    
  • 相关阅读:
    顺序容器2(用法)
    运算符重载
    IO相关3(string流)
    IO相关2(文件输入输出)
    C语言--指针
    Java---匿名类
    Andriod开发 --插件安装、环境配置、问题集锦
    ubuntu--vim 技巧
    ubuntu--命令大全
    各浏览器userAgent汇总
  • 原文地址:https://www.cnblogs.com/zswbky/p/8454106.html
Copyright © 2011-2022 走看看