什么是PyQuery
PyQuery是强大又灵活的网页解析库。如果熟悉jQuery的语法,那么PyQuery就是你的绝佳选择
安装PyQuery
pip3 install PyQuery
用法讲解
初始化
字符串初始化常用在得到网页源代码,然后进行解析。
html = """ <div> <ul> <li class="item-0">first item</li> <li class="item-1"><a href="link.html">second item</a></li> <li class="item-0 active"><a href="lin3.html"><span class="bold">third item</span></a></li> <li class="item-0"><a href="link5.html">fifth item</a></li> </ul> </div> """ from pyquery import PyQuery as pq doc = pq(html) print(doc('li'))#这里传入CSS选择器,标签选择器,结果返回所有li
URL初始化
from pyquery import PyQuery as pq doc = pq(url='http://www.baidu.com')#使用url初始化 print(doc('head'))#返回html中的head
文件初始化
from qyquery import PyQuery as pq doc = pq(filename='myhtml.html') print(doc('li'))
基本的CSS选择器
html = """ <div id="container"> <ul> <li class="item-0">first item</li> <li class="item-1"><a href="link.html">second item</a></li> <li class="item-0 active"><a href="lin3.html"><span class="bold">third item</span></a></li> <li class="item-0"><a href="link5.html">fifth item</a></li> </ul> </div> """ from pyquery import PyQuery as pq doc = pq(html) print(doc('#container .active a'))#查找id=container 下的类名为active的 下的所有a标签,只要有层级嵌套关系,不是直接父子关系。
查找元素
子元素,查找元素下的子元素
html = """ <div id="container"> <ul class="list"> <li class="item-0">first item</li> <li class="item-1"><a href="link.html">second item</a></li> <li class="item-0 active"><a href="lin3.html"><span class="bold">third item</span></a></li> <li class="item-0"><a href="link5.html">fifth item</a></li> </ul> </div> """ from pyquery import PyQuery as pq doc = pq(html) items = doc('.list') print(type(items)) print(items) lis = items.find('li')#想jQuery一样,可以用find筛选所有子元素,无论层级 print(type(lis)) print(lis)
返回结果: 返回值是<class 'pyquery.pyquery.PyQuery'>类型
<class 'pyquery.pyquery.PyQuery'>
<ul class="list">
<li class="item-0">first item</li>
<li class="item-1"><a href="link.html">second item</a></li>
<li class="item-0 active"><a href="lin3.html"><span class="bold">third item</span></a></li>
<li class="item-0"><a href="link5.html">fifth item</a></li>
</ul><class 'pyquery.pyquery.PyQuery'>
<li class="item-0">first item</li>
<li class="item-1"><a href="link.html">second item</a></li>
<li class="item-0 active"><a href="lin3.html"><span class="bold">third item</span></a></li>
<li class="item-0"><a href="link5.html">fifth item</a></li>
查找直接子元素
html = """ <div id="container"> <ul class="list"> <li class="item-0">first item</li> <li class="item-1"><a href="link.html">second item</a></li> <li class="item-0 active"><a href="lin3.html"><span class="bold">third item</span></a></li> <li class="item-0"><a href="link5.html">fifth item</a></li> </ul> </div> """ from pyquery import PyQuery as pq doc = pq(html) items = doc('.item-0')#可能返回多个 print(items.children())#返回直接子元素
返回值:
<a href="lin3.html"><span class="bold">third item</span></a><a href="link5.html">fifth item</a>
这里返回了2条class="item-0"的li的直接子元素
父元素
html = """ <div id="container"> <ul class="list"> <li class="item-0">first item</li> <li class="item-1"><a href="link.html">second item</a></li> <li class="item-0 active"><a href="lin3.html"><span class="bold">third item</span></a></li> <li class="item-0"><a href="link5.html">fifth item</a></li> </ul> </div> """ from pyquery import PyQuery as pq doc = pq(html) items = doc('.list') container = items.parent() print(type(container)) print(container)#返回整个class=list的父节点--div的内容
parents()--返回所有的祖先节点。
兄弟节点
html = """ <div id="container"> <ul class="list"> <li class="item-0">first item</li> <li class="item-1"><a href="link.html">second item</a></li> <li class="item-0 active"><a href="lin3.html"><span class="bold">third item</span></a></li> <li class="item-0"><a href="link5.html">fifth item</a></li> </ul> </div> """ from pyquery import PyQuery as pq doc = pq(html) items = doc('.active') print(items.siblings())#返回class=active的所有兄弟节点 print(items.siblings('.item-1'))#兄弟节点+CSS选择器
*返回结果:
<li class="item-1"><a href="link.html">second item</a></li>
<li class="item-0">first item</li>
<li class="item-0"><a href="link5.html">fifth item</a></li>
<li class="item-1"><a href="link.html">second item</a></li>
遍历
- 单个元素-返回的是单个元素那么就不谈遍历了,返回的是PyQuery对象
- 多个元素,如果返回的是多个元素就用.items(),那么返回的实际上是一个生成器,可以用for循环遍历
html = """ <div id="container"> <ul class="list"> <li class="item-0">first item</li> <li class="item-1"><a href="link.html">second item</a></li> <li class="item-0 active"><a href="lin3.html"><span class="bold">third item</span></a></li> <li class="item-0"><a href="link5.html">fifth item</a></li> </ul> </div> """ from pyquery import PyQuery as pq doc = pq(html) items = doc('.active') lis = items.siblings().items() print(type(lis)) for item in lis:#遍历所有兄弟节点 print(item)
获取信息
- 获取属性
html = """ <div id="container"> <ul class="list"> <li class="item-0">first item</li> <li class="item-1"><a href="link.html">second item</a></li> <li class="item-0 active"><a href="lin3.html"><span class="bold">third item</span></a></li> <li class="item-0"><a href="link5.html">fifth item</a></li> </ul> </div> """ from pyquery import PyQuery as pq doc = pq(html) a = doc('.item-0.active a') print(a) print(a.attr('href'))#返回属性为href的值 print(a.attr.href)#同上
返回结果:
<a href="lin3.html"><span class="bold">third item</span></a>
lin3.html
lin3.html
- 获取文本
html = """ <div id="container"> <ul class="list"> <li class="item-0">first item</li> <li class="item-1"><a href="link.html">second item</a></li> <li class="item-0 active"><a href="lin3.html"><span class="bold">third item</span></a></li> <li class="item-0"><a href="link5.html">fifth item</a></li> </ul> </div> """ from pyquery import PyQuery as pq doc = pq(html) a = doc('.item-0.active a') print(a) print(a.text())#返回a标签中的内容
返回结果:
<a href="lin3.html"><span class="bold">third item</span></a>
third item
- 获取HTML
html = """ <div id="container"> <ul class="list"> <li class="item-0">first item</li> <li class="item-1"><a href="link.html">second item</a></li> <li class="item-0 active"><a href="lin3.html"><span class="bold">third item</span></a></li> <li class="item-0"><a href="link5.html">fifth item</a></li> </ul> </div> """ from pyquery import PyQuery as pq doc = pq(html) a = doc('.item-0.active a') print(a) print(a.html())#返回整个a标签的html代码
返回结果:
<a href="lin3.html"><span class="bold">third item</span></a>
<span class="bold">third item</span>
DOM操作
- addClass、removeClass
html = """ <div id="container"> <ul class="list"> <li class="item-0">first item</li> <li class="item-1"><a href="link.html">second item</a></li> <li class="item-0 active"><a href="lin3.html"><span class="bold">third item</span></a></li> <li class="item-0"><a href="link5.html">fifth item</a></li> </ul> </div> """ from pyquery import PyQuery as pq doc = pq(html) li = doc('.item-0.active') print(li) li.removeClass('active')#删除这个li的类名active print(li) li.addClass('active')#又加上active类名 print(li)
输出结果
<li class="item-0 active"><a href="lin3.html"><span class="bold">third item</span></a></li>
<li class="item-0"><a href="lin3.html"><span class="bold">third item</span></a></li>
<li class="item-0 active"><a href="lin3.html"><span class="bold">third item</span></a></li>
- 添加或修改attr、css
html = """ <div id="container"> <ul class="list"> <li class="item-0">first item</li> <li class="item-1"><a href="link.html">second item</a></li> <li class="item-0 active"><a href="lin3.html"><span class="bold">third item</span></a></li> <li class="item-0"><a href="link5.html">fifth item</a></li> </ul> </div> """ from pyquery import PyQuery as pq doc = pq(html) li = doc('.item-0.active') print(li) li.attr('name','mylink')#因为没有name属性,所以实际上添加name属性,值为mylink print(li) li.css('font-size','20px')#添加行内style print(li)
- remove
html = """ <div class="wrap"> hello,lll <p>This is test</p> </div> """ from pyquery import PyQuery as pq doc = pq(html) wrap = doc('.wrap') print(wrap.text()) wrap.find('p').remove()#删除p标签 print(wrap.text())
- 其他DOM方法
https://pyquery.readthedocs.io/en/latest/api.html
- 伪类选择器
html = """ <div id="container"> <ul class="list"> <li class="item-0">first item</li> <li class="item-1"><a href="link.html">second item</a></li> <li class="item-0 active"><a href="lin3.html"><span class="bold">third item</span></a></li> <li class="item-0"><a href="link5.html">fifth item</a></li> </ul> </div> """ from pyquery import PyQuery as pq doc = pq(html) li = doc('li:first-child')#html标签下第一个li print(li) li = doc('li:last-child')#html标签下最后一个li print(li) li = doc('li:nth-child(2)')#html标签下第二个li print(li) li = doc('li:gt(2)')#html标签下大于2的所有li print(li) li = doc('li:nth-child(2n)')#html标签下偶数的li print(li) li = doc('li:contains(second)')#html标签下li中的内容包含second的li print(li)
- 更多的CSS选择器