爬虫基础：PyQuery

什么是PyQuery

PyQuery是强大又灵活的网页解析库。如果熟悉jQuery的语法，那么PyQuery就是你的绝佳选择

安装PyQuery

pip3 install PyQuery

用法讲解

初始化

字符串初始化常用在得到网页源代码，然后进行解析。

html = """
<div>
    <ul>
        <li class="item-0">first item</li>
        <li class="item-1"><a href="link.html">second item</a></li>
        <li class="item-0 active"><a href="lin3.html"><span class="bold">third item</span></a></li>
        <li class="item-0"><a href="link5.html">fifth item</a></li>
    </ul>
</div>
"""
from pyquery import PyQuery as pq
doc = pq(html)
print(doc('li'))#这里传入CSS选择器，标签选择器，结果返回所有li

URL初始化

from pyquery import PyQuery as pq
doc = pq(url='http://www.baidu.com')#使用url初始化
print(doc('head'))#返回html中的head

文件初始化

from qyquery import PyQuery as pq
doc = pq(filename='myhtml.html')
print(doc('li'))

基本的CSS选择器

html = """
<div id="container">
    <ul>
        <li class="item-0">first item</li>
        <li class="item-1"><a href="link.html">second item</a></li>
        <li class="item-0 active"><a href="lin3.html"><span class="bold">third item</span></a></li>
        <li class="item-0"><a href="link5.html">fifth item</a></li>
    </ul>
</div>
"""
from pyquery import PyQuery as pq
doc = pq(html)
print(doc('#container .active a'))#查找id=container 下的类名为active的 下的所有a标签，只要有层级嵌套关系，不是直接父子关系。

查找元素

子元素，查找元素下的子元素

html = """
<div id="container">
    <ul class="list">
        <li class="item-0">first item</li>
        <li class="item-1"><a href="link.html">second item</a></li>
        <li class="item-0 active"><a href="lin3.html"><span class="bold">third item</span></a></li>
        <li class="item-0"><a href="link5.html">fifth item</a></li>
    </ul>
</div>
"""
from pyquery import PyQuery as pq
doc = pq(html)
items = doc('.list')
print(type(items))
print(items)
lis = items.find('li')#想jQuery一样，可以用find筛选所有子元素，无论层级
print(type(lis))
print(lis)

返回结果: 返回值是<class 'pyquery.pyquery.PyQuery'>类型

<class 'pyquery.pyquery.PyQuery'>
<ul class="list">
<li class="item-0">first item</li>
<li class="item-1"><a href="link.html">second item</a></li>
<li class="item-0 active"><a href="lin3.html"><span class="bold">third item</span></a></li>
<li class="item-0"><a href="link5.html">fifth item</a></li>
</ul>

<class 'pyquery.pyquery.PyQuery'>
<li class="item-0">first item</li>
<li class="item-1"><a href="link.html">second item</a></li>
<li class="item-0 active"><a href="lin3.html"><span class="bold">third item</span></a></li>
<li class="item-0"><a href="link5.html">fifth item</a></li>

查找直接子元素

html = """
<div id="container">
    <ul class="list">
        <li class="item-0">first item</li>
        <li class="item-1"><a href="link.html">second item</a></li>
        <li class="item-0 active"><a href="lin3.html"><span class="bold">third item</span></a></li>
        <li class="item-0"><a href="link5.html">fifth item</a></li>
    </ul>
</div>
"""
from pyquery import PyQuery as pq
doc = pq(html)
items = doc('.item-0')#可能返回多个
print(items.children())#返回直接子元素

返回值：

<a href="lin3.html"><span class="bold">third item</span></a><a href="link5.html">fifth item</a>

这里返回了2条class="item-0"的li的直接子元素

父元素

html = """
<div id="container">
    <ul class="list">
        <li class="item-0">first item</li>
        <li class="item-1"><a href="link.html">second item</a></li>
        <li class="item-0 active"><a href="lin3.html"><span class="bold">third item</span></a></li>
        <li class="item-0"><a href="link5.html">fifth item</a></li>
    </ul>
</div>
"""
from pyquery import PyQuery as pq
doc = pq(html)
items = doc('.list')
container = items.parent()
print(type(container))
print(container)#返回整个class=list的父节点--div的内容

parents()--返回所有的祖先节点。

兄弟节点

html = """
<div id="container">
    <ul class="list">
        <li class="item-0">first item</li>
        <li class="item-1"><a href="link.html">second item</a></li>
        <li class="item-0 active"><a href="lin3.html"><span class="bold">third item</span></a></li>
        <li class="item-0"><a href="link5.html">fifth item</a></li>
    </ul>
</div>
"""
from pyquery import PyQuery as pq
doc = pq(html)
items = doc('.active')
print(items.siblings())#返回class=active的所有兄弟节点
print(items.siblings('.item-1'))#兄弟节点+CSS选择器

*返回结果:

<li class="item-1"><a href="link.html">second item</a></li>
<li class="item-0">first item</li>
<li class="item-0"><a href="link5.html">fifth item</a></li>
<li class="item-1"><a href="link.html">second item</a></li>

遍历

• 单个元素-返回的是单个元素那么就不谈遍历了，返回的是PyQuery对象
• 多个元素，如果返回的是多个元素就用.items()，那么返回的实际上是一个生成器，可以用for循环遍历

html = """
<div id="container">
    <ul class="list">
        <li class="item-0">first item</li>
        <li class="item-1"><a href="link.html">second item</a></li>
        <li class="item-0 active"><a href="lin3.html"><span class="bold">third item</span></a></li>
        <li class="item-0"><a href="link5.html">fifth item</a></li>
    </ul>
</div>
"""
from pyquery import PyQuery as pq
doc = pq(html)
items = doc('.active')
lis = items.siblings().items()
print(type(lis))
for item in lis:#遍历所有兄弟节点
    print(item)

获取信息

• 获取属性

html = """
<div id="container">
    <ul class="list">
        <li class="item-0">first item</li>
        <li class="item-1"><a href="link.html">second item</a></li>
        <li class="item-0 active"><a href="lin3.html"><span class="bold">third item</span></a></li>
        <li class="item-0"><a href="link5.html">fifth item</a></li>
    </ul>
</div>
"""
from pyquery import PyQuery as pq
doc = pq(html)
a = doc('.item-0.active a')
print(a)
print(a.attr('href'))#返回属性为href的值
print(a.attr.href)#同上

返回结果：

<a href="lin3.html"><span class="bold">third item</span></a>
lin3.html
lin3.html

• 获取文本

html = """
<div id="container">
    <ul class="list">
        <li class="item-0">first item</li>
        <li class="item-1"><a href="link.html">second item</a></li>
        <li class="item-0 active"><a href="lin3.html"><span class="bold">third item</span></a></li>
        <li class="item-0"><a href="link5.html">fifth item</a></li>
    </ul>
</div>
"""
from pyquery import PyQuery as pq
doc = pq(html)
a = doc('.item-0.active a')
print(a)
print(a.text())#返回a标签中的内容

返回结果：

<a href="lin3.html"><span class="bold">third item</span></a>
third item

• 获取HTML

html = """
<div id="container">
    <ul class="list">
        <li class="item-0">first item</li>
        <li class="item-1"><a href="link.html">second item</a></li>
        <li class="item-0 active"><a href="lin3.html"><span class="bold">third item</span></a></li>
        <li class="item-0"><a href="link5.html">fifth item</a></li>
    </ul>
</div>
"""
from pyquery import PyQuery as pq
doc = pq(html)
a = doc('.item-0.active a')
print(a)
print(a.html())#返回整个a标签的html代码

返回结果:

<a href="lin3.html"><span class="bold">third item</span></a>
<span class="bold">third item</span>

DOM操作

• addClass、removeClass

html = """
<div id="container">
    <ul class="list">
        <li class="item-0">first item</li>
        <li class="item-1"><a href="link.html">second item</a></li>
        <li class="item-0 active"><a href="lin3.html"><span class="bold">third item</span></a></li>
        <li class="item-0"><a href="link5.html">fifth item</a></li>
    </ul>
</div>
"""
from pyquery import PyQuery as pq
doc = pq(html)
li = doc('.item-0.active')
print(li)
li.removeClass('active')#删除这个li的类名active
print(li)
li.addClass('active')#又加上active类名
print(li)

输出结果

<li class="item-0 active"><a href="lin3.html"><span class="bold">third item</span></a></li>
<li class="item-0"><a href="lin3.html"><span class="bold">third item</span></a></li>
<li class="item-0 active"><a href="lin3.html"><span class="bold">third item</span></a></li>

• 添加或修改attr、css

html = """
<div id="container">
    <ul class="list">
        <li class="item-0">first item</li>
        <li class="item-1"><a href="link.html">second item</a></li>
        <li class="item-0 active"><a href="lin3.html"><span class="bold">third item</span></a></li>
        <li class="item-0"><a href="link5.html">fifth item</a></li>
    </ul>
</div>
"""
from pyquery import PyQuery as pq
doc = pq(html)
li = doc('.item-0.active')
print(li)
li.attr('name','mylink')#因为没有name属性，所以实际上添加name属性，值为mylink
print(li)
li.css('font-size','20px')#添加行内style
print(li)

• remove

html = """
<div class="wrap">
    hello,lll
    <p>This is test</p>
</div>
"""
from pyquery import PyQuery as pq
doc = pq(html)
wrap = doc('.wrap')
print(wrap.text())
wrap.find('p').remove()#删除p标签
print(wrap.text())

• 其他DOM方法

https://pyquery.readthedocs.io/en/latest/api.html

• 伪类选择器

html = """
<div id="container">
    <ul class="list">
        <li class="item-0">first item</li>
        <li class="item-1"><a href="link.html">second item</a></li>
        <li class="item-0 active"><a href="lin3.html"><span class="bold">third item</span></a></li>
        <li class="item-0"><a href="link5.html">fifth item</a></li>
    </ul>
</div>
"""
from pyquery import PyQuery as pq
doc = pq(html)
li = doc('li:first-child')#html标签下第一个li
print(li)
li = doc('li:last-child')#html标签下最后一个li
print(li)
li = doc('li:nth-child(2)')#html标签下第二个li
print(li)
li = doc('li:gt(2)')#html标签下大于2的所有li
print(li)
li = doc('li:nth-child(2n)')#html标签下偶数的li
print(li)
li = doc('li:contains(second)')#html标签下li中的内容包含second的li
print(li)

• 更多的CSS选择器

http://www.w3school.com.cn/css/index.asp