day_5：动态渲染页面爬取

一、Selenium

1、声明浏览器对象

from selenium import webdriver

browser_chrome = webdriver.Chrome()
browser_firefox = webdriver.Firefox()
browser_edge = webdriver.Edge()
browser_phantomjs = webdriver.PhantomJS()
browser_safari = webdriver.safari()

2、访问页面

from selenium import webdriver

browser = webdriver.Chrome()
browser.get('https://www.taobao.com')
print(browser.page_source)
browser.close()

3、查找节点

# 定位单个节点
find_element_by_id
find_element_by_name
find_element_by_xpath
find_element_by_link_text #通过精确文本定位
find_element_by_partial_link_text #通过模糊文本定位
find_element_by_tag_name
find_element_by_class_name
find_element_by_css_selector

# 定位多个节点
find_elements_by_name
find_elements_by_xpath
find_elements_by_link_text
find_elements_by_partial_link_text
find_elements_by_tag_name
find_elements_by_class_name
find_elements_by_css_selector

# 公共方法
find_element(By.x, 'x')
find_elements(By.x, 'x')

#By类型
ID = "id"
XPATH = "xpath"
LINK_TEXT = "link text"
PARTIAL_LINK_TEXT = "partial link text"
NAME = "name"
TAG_NAME = "tag name"
CLASS_NAME = "class name"
CSS_SELECTOR = "css selector"

# 4中方法查找淘宝首页搜索框节点
from selenium import webdriver

browser = webdriver.Chrome()
browser.get('https://www.taobao.com')
input_id = browser.find_element_by_id('q')
input_name = browser.find_element_by_name('q')
input_css = browser.find_element_by_css_selector('#q')
input_xpath = browser.find_element_by_xpath('//*[@id="q"]')
print(input_id)
print(input_name)
print(input_css)
print(input_xpath)
browser.close()

from selenium import webdriver
from selenium.webdriver.common.by import By

browser = webdriver.Chrome()
browser.get('https://www.taobao.com')

# find_element(查找方式By，查找值)，以下两种方式等价
input_first = browser.find_element(By.ID, 'q')
input_second = browser.find_element_by_id('q')
print(input_first)
print(input_second)
browser.close()

from selenium import webdriver

browser = webdriver.Chrome()
browser.get('https://www.taobao.com')

# 查找淘宝左侧导航栏所有li标签
lis = browser.find_elements_by_css_selector('.service-bd li')
for li in lis:
    print(li)
browser.close()

from selenium import webdriver

browser = webdriver.Chrome()

browser.get('https://www.zhihu.com/explore')
logo = browser.find_element_by_id('zh-top-link-logo')
print(logo)
# get_attribute(属性名称)获取节点属性
print(logo.get_attribute('class'))
print(logo.get_attribute('href'))
print(logo.get_attribute('id'))
print(logo.get_attribute('data-za-c'))

# logo.text 获取节点文本
print(logo.text)
print(logo.id)

# logo.location 获取节点位置
print(logo.location)
print(logo.parent)
print(logo.rect)

# logo.tag_name 获取节点名称
print(logo.tag_name)
print(logo.size)
browser.close()

4、节点交互

# 淘宝搜索商品
from selenium import webdriver
import time

browser = webdriver.Chrome()
browser.get('https://www.taobao.com')

# 查找输入框
input = browser.find_element_by_id('q')

# 在输入框中输入
input.send_keys('iPhone') 
time.sleep(1)

# 清空输入框 
input.clear()  

# 在输入框中输入
input.send_keys('iPad')  

# 查找搜索按钮
btn = browser.find_element_by_css_selector('.btn-search')

# 点击按钮
btn.click()  

5、动作链（鼠标滑动、拖拽，键盘按键等）

# select下拉框
rom selenium.webdriver.support.ui import Select
select = Select(driver.find_element_by_name('name'))
select.select_by_index(index)  # 通过索引定位
select.select_by_visible_text("text") # 通过文本定位
select.select_by_value(value) # 通过值定位

select = Select(driver.find_element_by_id('id'))
select.deselect_all() # 取消所有选择

# 拖放：将元素移动一定量，或者移动到另一个元素

from selenium.webdriver import ActionChains

element = driver.find_element_by_name("source")
target = driver.find_element_by_name("target")

action_chains = ActionChains(driver)
action_chains.drag_and_drop(element, target).perform()

# cookies

driver.get("http://www.example.com")

cookie = {'name' : 'foo', 'value': 'bar'}
driver.add_cookie(cookie)

driver.get_cookies()

6、等待（页面加载过慢导致找不到节点，需要等待页面加载完成再找节点）显式等待设置的是最长等待时间，尽量使用显示等待、显等待待设置的是固定的等待时间

# 显式等待
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

driver = webdriver.Firefox()
driver.get("http://somedomain/url_that_delays_loading")
try:
    element = WebDriverWait(driver, 10).until(
        EC.presence_of_element_located((By.ID, "myDynamicElement"))
    )
finally:
    driver.quit()


# 隐式等待
from selenium import webdriver

driver = webdriver.Firefox()
driver.implicitly_wait(10) # seconds
driver.get("http://somedomain/url_that_delays_loading")
myDynamicElement = driver.find_element_by_id("myDynamicElement")

7、Headless模式（无界面模式）

from selenium import webdriver

chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--headless')
driver = webdriver.Chrome(chrome_options=chrome_options)

driver.get('https://www.taobao.com')
print(driver.page_source)