python3 爬取知乎 模拟登录

#!/usr/bin/env python
# -*- coding: utf-8 -*-
import requests
try:
    import cookielib

except:
    import http.cookiejar as cookielib
import re
import time
import os.path
try:
    from PIL import Image
except:
    pass
from bs4 import BeautifulSoup


# 构造 Request headers
#从配置表获取
# agent = 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3294.6 Safari/537.36'
#
# headers = {
#
#     "Host": "www.zhihu.com",
#
#     "Referer": "https://www.zhihu.com/",
#
#     'User-Agent': agent
#
# }

cookie_file = 'cookie.txt'
header={
    'Host': 'www.zhihu.com',
    'Referer': 'http://www.zhihu.com',
    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.116 Safari/537.36',
    }
# 使用登录cookie信息

session = requests.session()

session.cookies = cookielib.LWPCookieJar(filename=cookie_file)

try:

    session.cookies.load(ignore_discard=True)

except:

    print("Cookie 未能加载")



def get_xsrf():

    '''_xsrf 是一个动态变化的参数'''

    index_url = 'https://www.zhihu.com'

    # 获取登录时需要用到的_xsrf

    index_page = session.get(index_url, headers=header)

    # html = index_page.cookies

    # pattern = r'name="_xsrf" value="(.*?)"'
    #
    # # 这里的_xsrf 返回的是一个list
    #
    # _xsrf = re.findall(pattern, html)
    xsrf = index_page.request._cookies.get("_xsrf")
    return xsrf



# 获取验证码

def get_captcha():

    t = str(int(time.time() * 1000))

    captcha_url = 'https://www.zhihu.com/captcha.gif?r=' + t + "&type=login"

    r = session.get(captcha_url, headers=header)

    with open('captcha.jpg', 'wb') as f:

        f.write(r.content)

        f.close()

    # 用pillow 的 Image 显示验证码

    # 如果没有安装 pillow 到源代码所在的目录去找到验证码然后手动输入

    try:

        im = Image.open('captcha.jpg')

        im.show()

        im.close()

    except:

        print(u'请到 %s 目录找到captcha.jpg 手动输入' % os.path.abspath('captcha.jpg'))

    captcha = input("please input the captcha
>")

    return captcha



def isLogin():

    # 通过查看用户个人信息来判断是否已经登录

    url = "https://www.zhihu.com/settings/profile"

    login_code = session.get(url, headers=header, allow_redirects=False).status_code

    if login_code == 200:

        return True

    else:

        return False



def login(secret, account):

    # 通过输入的用户名判断是否是手机号

    if re.match(r"^1d{10}$", account):

        print("手机号登录 
")

        post_url = 'https://www.zhihu.com/login/phone_num'

        postdata = {
            '_xsrf': get_xsrf(),

            'password': secret,

            'remember_me': 'true',

            'phone_num': account,

        }

    else:

        if "@" in account:

            print("邮箱登录 
")

        else:

            print("你的账号输入有问题，请重新登录")

            return 0

        post_url = 'https://www.zhihu.com/login/email'

        postdata = {

            '_xsrf': get_xsrf(),

            'password': secret,

            'remember_me': 'true',

            'email': account,

        }

    try:

        # 不需要验证码直接登录成功

        login_page = session.post(post_url, data=postdata, headers=header)

        login_code = login_page.text

        print(login_page.status_code)

        print(login_code)

    except:

        # 需要输入验证码后才能登录成功

        postdata["captcha"] = get_captcha()

        login_page = session.post(post_url, data=postdata, headers=header)

        login_code = eval(login_page.text)

        print(login_code['msg'])
    cookie_path=cookie_file
    session.cookies.save(cookie_path)
# try:
#
#     input = raw_input
#
# except:
#
#     pass



##将主页面的用户提问print到shell上

def  getpage(url2):

  mainpage = session.get(url2, headers=header)

  soup=BeautifulSoup(mainpage.text,'html.parser')

  tags=soup.find_all("a",class_="question_link")

  #print tags



  for tag in tags:

    print (tag.string)


def get_login_cookie(url):
    '''
    获取保存cookie
    :param url:
    :return:
    '''
    
    if not os.path.exists(cookie_file):
        account = input('请输入你的用户名
>  ')
        secret = input("请输入你的密码
>  ")
        user_name = account
        passwd = secret
        login(passwd, user_name)
    try:
        cookie_jar = cookielib.LWPCookieJar(cookie_file)
        cookie_jar.load(ignore_discard=True, ignore_expires=True)
        print ('Load cookie succeeded')
    except cookielib.LoadError:
        return None
    else:
        cookie_d = {}
        for cookie in cookie_jar:
            domain = cookie.domain
            if url.find(domain) > 0:
                cookie_d[cookie.name] = cookie.value
        return cookie_d


if __name__ == '__main__':

    if isLogin():

        print('您已经登录')

        url2='https://www.zhihu.com'

        getpage(url2)

    else:

        account = input('请输入你的用户名
>  ')

        secret = input("请输入你的密码
>  ")

        login(secret, account)

import json
import pickle
import time

from selenium import webdriver
#代码中的chrome_options.add_argument()非常关键，一是要以无界面形式运行，二是禁用沙盒，否则程序报错。
#linux上使用,打开方式以无界面打开，windows不需要这些
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--headless') # 指定无界面形式运行
chrome_options.add_argument('no-sandbox') # 禁止沙盒


tianyan_cookie_file = 'tianyancha_cookie.txt'
qichacha_cookie_file = 'qichacha_cookie.txt'

#注:这些方法还需要一个插件  下面 chromedriver.exe ,也可以用火狐
# 同目录下(脚本运行目录)放置  chromedriver.exe  这个文件, 下载地址http://npm.taobao.org/mirrors/chromedriver/
# 部署的话最好部署在windows服务器下,安装最新的谷歌浏览器,linux也可以使用,但是不能使用界面操作

#天眼查账号登录
def login(url):
    # dirver.maximize_window()
    dirver = webdriver.Chrome(chrome_options=chrome_options)
    dirver.get(url)
    time.sleep(3)
    # page_source = dirver.page_source
    # print(page_source)
    phoneNumber = dirver.find_elements_by_xpath('//*[@id="web-content"]/div/div/div/div[2]/div/div[2]/div[2]/div[2]/div[2]/input')
    phone = input("你的电话")
    phoneNumber[0].send_keys(phone)

    password = dirver.find_elements_by_xpath('//*[@id="web-content"]/div/div/div/div[2]/div/div[2]/div[2]/div[2]/div[3]/input')
    # 密码
    passwords = input("你的密码")
    password[0].send_keys(passwords)

    time.sleep(3)

    sublit = dirver.find_elements_by_xpath('//*[@id="web-content"]/div/div/div/div[2]/div/div[2]/div[2]/div[2]/div[5]')
    sublit[0].click()
    time.sleep(3)
    print(dirver.get_cookies())
    print(type(dirver.get_cookies()))

    with open(tianyan_cookie_file, 'wb') as handle:
        pickle.dump(dirver.get_cookies(), handle)
    time.sleep(60)
    # dirver.quit()
#使用微博模拟登陆
def login_qichacha(url):
    dirver = webdriver.Chrome(chrome_options=chrome_options)
    dirver.get(url)
    time.sleep(3)

    weibologin = dirver.find_elements_by_xpath('//*[@id="qrcodeLoginPanel"]/div[2]/div/div[3]/a[3]')
    weibologin[0].click()
    time.sleep(5)
    #用户名  或者微博名
    phone = input("你的电话")
    dirver.find_elements_by_xpath('//*[@id="userId"]')[0].send_keys(phone)
    time.sleep(3)
    #密码
    password = input("你的微博密码")
    dirver.find_elements_by_xpath('//*[@id="passwd"]')[0].send_keys(password)
    time.sleep(3)
    sublimt = dirver.find_elements_by_xpath('//*[@id="outer"]/div/div[2]/form/div/div[2]/div/p/a[1]')
    sublimt[0].click()
    time.sleep(10)
    print(dirver.get_cookies())
    print(type(dirver.get_cookies()))

    with open(qichacha_cookie_file, 'wb') as handle:
        pickle.dump(dirver.get_cookies(), handle)
    dirver.close()
    # time.sleep(60)
def getCookieStr():
    #打开文件
    with open(qichacha_cookie_file, 'rb') as handle:
        b = pickle.load(handle)
        print(b)
        cookie_str = ''
        for index in b:
            cookie_str += index['name'] + '=' + index['value'] + ';'
        cookie_str = cookie_str[:-1]
        print(cookie_str)
    return cookie_str



if __name__ == '__main__':
    #天眼查
    # url = 'https://www.tianyancha.com/login'
    # login(url)

    # 企查查
    url = 'https://www.qichacha.com/user_login'
    login_qichacha(url)

    #保存cookie
    # with open(cookie_file, 'wb') as handle:
    #     pickle.dump(data, handle)
    # #打开文件
    # with open(qichacha_cookie_file, 'rb') as handle:
    #     b = pickle.load(handle)
    #     print(b)
    #     cookie_str = ''
    #     for index in b:
    #         cookie_str += index['name'] + '=' + index['value'] + ';'
    #     cookie_str = cookie_str[:-1]
    #     print(cookie_str)

 大家注意linux环境下(centos7.0下)安装依赖参考

安装最新版本google-chrome

yum install https://dl.google.com/linux/direct/google-chrome-stable_current_x86_64.rpm
1
测试安装

google-chrome-stable --headless --disable-gpu --screenshot http://www.baidu.com/
1
报错：

[1026/154358.043744:ERROR:zygote_host_impl_linux.cc(89)] Running as
root without --no-sandbox is not supported. See
https://crbug.com/638180.

需要指定以no-sandbox方式运行。

google-chrome-stable --no-sandbox --headless --disable-gpu --screenshot http://www.baidu.com/
1
Fontconfig warning: “/etc/fonts/fonts.conf”, line 86: unknown element “blank”
[1026/154710.301159:ERROR:gpu_process_transport_factory.cc(980)] Lost UI shared context.
[1026/154711.126035:INFO:headless_shell.cc(538)] Written to file screenshot.png.

安装chrome-driver

wget http://npm.taobao.org/mirrors/chromedriver/70.0.3538.67/chromedriver_linux64.zip
这个是解压zip 到指定文件夹
      tar -zcvf chromedriver_linux64.zip  /usr/local/bin/
或  (二选一) 
unzip -d /usr/local/bin chromedriver_linux64.zip
1
2
python3脚本

代码中的chrome_options.add_argument()非常关键，一是要以无界面形式运行，二是禁用沙盒，否则程序报错。

# -*- coding: utf-8 -*-
import time
from selenium import webdriver

chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--headless') # 指定无界面形式运行
chrome_options.add_argument('no-sandbox') # 禁止沙盒
driver = webdriver.Chrome(chrome_options=chrome_options)

driver.get('http://www.baidu.com/')
time.sleep(10)
print(driver.page_source) 

driver.close() 
driver.quit() 
---------------------