如何巧妙的利用selenium和requests组合来进行操作需要登录的页面

一、在这里selenium的作用

（1）模拟的登录。

（2）获取登录成功之后的cookies

代码

    def start_login(self):
        chrome_options = Options()
        # 禁止图片加载,禁止推送通知
        prefs = {
            "profile.default_content_setting_values": {
                "images": 2
            }, "profile.default_content_setting_values.notifications": 2
        }
        chrome_options.add_experimental_option("prefs", prefs)
        if chrome_args().get("headless_flag") == "1":
            chrome_options.add_argument(chrome_args().get("headless"))
        chrome_options.add_argument(chrome_args().get("nogpu"))
        chrome_options.add_argument(chrome_args().get("noinfobars"))
        chrome_options.add_argument(chrome_args().get("max_windows"))
        chrome_options.add_argument(self.Proxy_server)

        driver = webdriver.Chrome(chrome_options=chrome_options)
        try:
            get_logger().info("start login.....")
            try:
                # login info
                self.login_name = self.loginInfo.get("login_id")
                self.password = aes_cbc_decrypt(self.loginInfo.get("login_pwd"))
            except Exception:
                get_logger().error("cant get login info,here are detals".format(traceback.format_exc()))
            wait = WebDriverWait(driver, 30)
            print(u"start login in")
            driver.get(self.login_url)
            try:
                login_id = wait.until(
                    eccd.presence_of_element_located(
                        (By.XPATH, self.Id_xpath))
                )
                login_id.send_keys(self.login_name)
                login_id.send_keys(Keys.ENTER)
                password = wait.until(
                    eccd.presence_of_element_located(
                        (By.XPATH, self.pwd_xpath))
                )
                password.send_keys(self.password)
                submit = wait.until(
                    eccd.presence_of_element_located(
                        (By.XPATH, self.login_btn_xpath))
                )
                submit.click()
                # login signal
                #判断是否显示 右上角是否显示用户名
                login_ok = wait.until(
                    eccd.presence_of_element_located(
                        (By.XPATH, self.login_ok_xpath))
                )
                try:
                    #判断登录成功
                    if login_ok:
                        get_logger().info("get user name successful:"+u"{}".format(login_ok.text))
                        try:
                            get_weibo_info=driver.find_element_by_xpath(self.forward_home_page)
                        except:

                            driver.get(self.first_page)

                        forward_home_page_ok = wait.until(
                              eccd.presence_of_element_located(
                                  (By.XPATH, self.forward_home_page))
                          )
                        forward_home_page_ok.click()
                        time.sleep(5)
                        self.first_page=driver.current_url
                        get_logger().info("get homepage successful,url is {}".format(driver.current_url))
                        cookies = driver.get_cookies()  # 导出cookie
                        get_logger().info("get cookies")
                        get_fansnum_ok= wait.until(
                                eccd.presence_of_element_located(
                                    (By.XPATH, self.follow_xpath))
                        )
                        self.get_followfanshome(driver)
                        get_logger().info("get_followfanshome ok")
                        self.home_page_source=driver.page_source
                        driver.close()  # 关闭chrome
                        #获取主页的
                        # 如果需要保存cookies
                        self.write_cookie(cookies)
                        get_logger().info("get cookies,login ok")
                        return cookies


                    else:
                        raise RuntimeError('login failed')
                except:
                    get_logger().error("login failed")
                    return None

            except:
                if driver is not None:
                    driver.close()
        except Exception as e:
            if driver is not None:
                driver.close()
                get_logger().error("dbs operation error，here are details:{}".format(traceback.format_exc()))

　通过使用cookies = driver.get_cookies() #我们获取了cookie那么我们怎用呢。

二、requests如何使用获取到的cookies

1.创建一个Session对象

req = requests.Session()  # 构建Session

2.转换上面的cookies对象

for cookie in cookies:
            req.cookies.set(cookie['name'], cookie['value'])  # 转换cookies

3.开始访问该网站的需求页面

data = req.get(url, headers=self.headers).text #获取文本网页
jsondata = req.get(url, headers=self.headers).json() #获取json型网页

　　好了就是这么简单，另外补充一个经常使用的库fake-useragent，可以获取随机的useragent，但是其中谷歌的useragent版本比较低，尤其在访问知乎的时候，会提示浏览器版本低的信息，这一点需要注意。

另外拥有一个headers是对爬虫起码的尊重，所以有个构建好的headers是非常必要的。我就经常用下面的例：

from fake_useragent import UserAgent as UA
import random

headers = {
            'Accept': 'text/html, application/xhtml+xml, image/jxr, */*',
            'Accept - Encoding': 'gzip, deflate',
            'Accept-Language': 'zh-Hans-CN, zh-Hans; q=0.5',
            'Connection': 'Keep-Alive',
            'User-Agent': UA().random #获取随机的useragent
        }