1 # -*- coding:utf-8 -*- 2 ''' 3 Created on Sep 10, 2018 4 5 @author: SaShuangYiBing 6 ''' 7 from lxml import etree 8 9 html=''' 10 <html> 11 <head> 12 <title>哈哈测试一下</title> 13 <link type="text/css" rel="stylesheet" href="haha.css" /> 14 <link type="text/css" rel="stylesheet" href="haha1.css" /> 15 <link type="text/css" rel="stylesheet" href="haha2.css" /> 16 <script type="text/javascript" src="haha.js"></script> 17 <script type="text/javascript" src="haha1.js"></script> 18 <script type="text/javascript" src="haha2.js"></script> 19 </head> 20 <body> 21 <div id="id1" class="class1"> 22 <div id="id2" class="class2"> 23 <ul class="cls_ul1"> 24 <li class="cls_li1"> 25 <div class="cls_3"> 26 <span>span_text1</span> 27 <span>span_text2</span> 28 <i>text_1</i> 29 </div> 30 <div> 31 <a href="a_1.html">a_1</a> 32 <a href="a_2.html">a_2</a> 33 <a href="a_3.html">a_3</a> 34 </div> 35 <div class="cls_4"> 36 <a href="a_4.html"> 37 <img href="a_img1.jpg" /> 38 </a> 39 </div> 40 </li> 41 <li class="cls_li1"> 42 <div class="cls_3"> 43 <span>span_text3</span> 44 <span>span_text4</span> 45 <i>text_2</i> 46 <i>text_22</i> 47 </div> 48 <div> 49 <a href="a_4.html">a_4</a> 50 <a href="a_5.html">a_5</a> 51 <a href="a_6.html">a_6</a> 52 </div> 53 <div class="cls_4"> 54 <a href="a_5.html"> 55 <img href="a_img2.jpg" /> 56 </a> 57 </div> 58 </li> 59 </ul> 60 </div> 61 <div id="id3" class="class3"> 62 <ul class="cls_ul2"> 63 <li class="cls_li2"> 64 <div class="cls_5"> 65 <span>span_text5</span> 66 <span>span_text6</span> 67 <i>text_3</i> 68 </div> 69 <div> 70 <a href="a_1.html">a_1</a> 71 <a href="a_2.html">a_2</a> 72 <a href="a_3.html">a_3</a> 73 </div> 74 <div class="cls_6"> 75 <a href="a_4.html"> 76 <img href="a_img3.jpg" /> 77 </a> 78 </div> 79 </li> 80 <li class="cls_li2"> 81 <div class="cls_5"> 82 <span>span_text7</span> 83 <span>span_text8</span> 84 <i>text_4</i> 85 </div> 86 <div> 87 <a href="a_4.html">a_4</a> 88 <a href="a_5.html">a_5</a> 89 <a href="a_6.html">a_6</a> 90 </div> 91 <div class="cls_6"> 92 <a href="a_5.html"> 93 <img href="a_img4.jpg" /> 94 </a> 95 </div> 96 </li> 97 </ul> 98 </div> 99 </div> 100 </body> 101 </html> 102 ''' 103 104 html_data = etree.HTML(html) 105 106 # 1、从根节点开始，沿着XML路径一步一步选择节点，text()表示节点内容 107 content = html_data.xpath("/html/head/title/text()") 108 for con in content: 109 print (con) 110 print ("~~~~~~~~~这是第一个分隔线~~~~~~~~~") 111 112 # 2、从根节点开始，沿着XML路径一步一步选择节点，text表示节点内容 113 nodes = html_data.xpath("/html/head/title") 114 for i in nodes: 115 print (i.text) 116 print ("~~~~~~~~~这是第二个分隔线~~~~~~~~~") 117 118 # 3、从文档中某个节点开始，不考虑此节点位置，text()表示节点内容 119 content = html_data.xpath("//title/text()") 120 for con in content: 121 print (con) 122 print ("~~~~~~~~~这是第三个分隔线~~~~~~~~~") 123 124 # 4、获取所有div(html/body/div/div)的id属性值 125 nodes = html_data.xpath("/html/body/div/div") 126 for i in range(len(nodes)): 127 content = nodes[i].xpath("@id") 128 for con in content: 129 print (con) 130 print ("~~~~~~~~~这是第四个分隔线~~~~~~~~~") 131 132 # 5、body节点下某节点的属性值 133 content = html_data.xpath("body/div/div[@id= 'id2']/ul/li[1]/div[2]/a/@href") 134 for con in content: 135 print (con) 136 print ("~~~~~~~~~这是第五个分隔线~~~~~~~~~") 137 138 # 6、div[@id='id2']节点下某节点的属性值 139 content = html_data.xpath("//div[@id = 'id2']/ul/li[1]/div[2]/a/@href") 140 for con in content: 141 print (con) 142 print ("~~~~~~~~~这是第六个分隔线~~~~~~~~~") 143 144 # 7、div[@id='id2']节点下某节点的内容 145 content = html_data.xpath("//div[@id= 'id2']/ul/li[1]/div[2]/a/text()") 146 for con in content: 147 print (con) 148 print ("~~~~~~~~~这是第七个分隔线~~~~~~~~~") 149 150 # 8、用'*'来匹配任何元素 151 content = html_data.xpath("*//div[@id = 'id2']/ul/li[1]/div[2]/a/text()") 152 for con in content: 153 print (con) 154 print ("~~~~~~~~~这是第八个分隔线~~~~~~~~~") 155 156 # 9、选取多个节点 157 nodes = html_data.xpath("//i|//span") 158 for i in range(len(nodes)): 159 print (nodes[i].text) 160 print ("~~~~~~~~~这是第九个分隔线~~~~~~~~~") 161 162 # 10、选取所有li节点 163 nodes = html_data.xpath("//li") 164 for i in range(len(nodes)): 165 content = nodes[i].xpath("div/@class") # li节点下所有div节点的class属性值 166 print (i,'='*5) 167 for con in content: 168 print (con) 169 print ("~~~~~~~~~这是第十个分隔线~~~~~~~~~") 170 171 # 11、选取所有li节点 172 nodes = html_data.xpath("//li") 173 for i in range(len(nodes)): 174 content = nodes[i].xpath("div[last()]/@class") # li节点下最后一个div节点的class属性值 175 print (i, '='*5) 176 for con in content: 177 print (con) 178 print ("~~~~~~~~~这是第十一个分隔线~~~~~~~~~") 179 180 # 12、这里应用了'..'和'@'，其中'..'表示父节点，具体就是上一步(title)的父节点head；'@'表示属性，就是它后面接是属性名，在这里的意思就是属性href的内容 181 content = html_data.xpath("/html/head/title/../script/@src") 182 for con in content: 183 print (con) 184 print ("~~~~~~~~~这是第十二个分隔线~~~~~~~~~") 185 186 # 13、div[@class='cls_3']的子节点span的兄弟节点i 187 nodes = html_data.xpath("//div[@class = 'cls_3']/span/following-sibling::i") 188 for i in range(len(nodes)): 189 content = nodes[i].xpath("./text()") # 当前节点内容 190 for con in content: 191 print (con) 192 print ("~~~~~~~~~这是第十三个分隔线~~~~~~~~~") 193 194 # 14、li[@class='cls_li1']后代节点里第一个div的class属性值 195 content = html_data.xpath("//li[@class = 'cls_li1']/descendant::div[1]/@class") 196 for con in content: 197 print (con) 198 print ("~~~~~~~~~这是第十四个分隔线~~~~~~~~~") 199 200 201 # 15、li[@class='cls_li1']后代节点里span的内容 202 content = html_data.xpath("//li[@class = 'cls_li1']/descendant::span/text()") 203 for con in content: 204 print (con) 205 print ("~~~~~~~~~这是第十五个分隔线~~~~~~~~~") 206 207 # 16、用'*'来匹配任何元素，且不包含class属性的div节点 208 content = html_data.xpath("*//div[@id = 'id2']/ul/li[1]/div[not(@class)]/a/text()") 209 for con in content: 210 print (con) 211 print ("~~~~~~~~~这是第十六个分隔线~~~~~~~~~") 212 213 # 17、多个条件的情况 214 content = html_data.xpath("//div[@id= 'id2' and @class= 'class2']/ul/li[1]/div[1]/span/text()") 215 for con in content: 216 print (con) 217 print ("~~~~~~~~~这是第十七个分隔线~~~~~~~~~") 218 219 # 18、contains 包含的情况 220 content = html_data.xpath("//div[contains(@class,'class2')]/ul/li[2]/div[2]/a/@href") 221 for con in content: 222 print (con) 223 print ("~~~~~~~~~这是第十八个分隔线~~~~~~~~~") 224 225 输出如下： 226 227 哈哈测试一下 228 ~~~~~~~~~这是第一个分隔线~~~~~~~~~ 229 哈哈测试一下 230 ~~~~~~~~~这是第二个分隔线~~~~~~~~~ 231 哈哈测试一下 232 ~~~~~~~~~这是第三个分隔线~~~~~~~~~ 233 id2 234 id3 235 ~~~~~~~~~这是第四个分隔线~~~~~~~~~ 236 a_1.html 237 a_2.html 238 a_3.html 239 ~~~~~~~~~这是第五个分隔线~~~~~~~~~ 240 a_1.html 241 a_2.html 242 a_3.html 243 ~~~~~~~~~这是第六个分隔线~~~~~~~~~ 244 a_1 245 a_2 246 a_3 247 ~~~~~~~~~这是第七个分隔线~~~~~~~~~ 248 a_1 249 a_2 250 a_3 251 ~~~~~~~~~这是第八个分隔线~~~~~~~~~ 252 span_text1 253 span_text2 254 text_1 255 span_text3 256 span_text4 257 text_2 258 text_22 259 span_text5 260 span_text6 261 text_3 262 span_text7 263 span_text8 264 text_4 265 ~~~~~~~~~这是第九个分隔线~~~~~~~~~ 266 0 ===== 267 cls_3 268 cls_4 269 1 ===== 270 cls_3 271 cls_4 272 2 ===== 273 cls_5 274 cls_6 275 3 ===== 276 cls_5 277 cls_6 278 ~~~~~~~~~这是第十个分隔线~~~~~~~~~ 279 0 ===== 280 cls_4 281 1 ===== 282 cls_4 283 2 ===== 284 cls_6 285 3 ===== 286 cls_6 287 ~~~~~~~~~这是第十一个分隔线~~~~~~~~~ 288 haha.js 289 haha1.js 290 haha2.js 291 ~~~~~~~~~这是第十二个分隔线~~~~~~~~~ 292 text_1 293 text_2 294 text_22 295 ~~~~~~~~~这是第十三个分隔线~~~~~~~~~ 296 cls_3 297 cls_3 298 ~~~~~~~~~这是第十四个分隔线~~~~~~~~~ 299 span_text1 300 span_text2 301 span_text3 302 span_text4 303 ~~~~~~~~~这是第十五个分隔线~~~~~~~~~ 304 a_1 305 a_2 306 a_3 307 ~~~~~~~~~这是第十六个分隔线~~~~~~~~~ 308 span_text1 309 span_text2 310 ~~~~~~~~~这是第十七个分隔线~~~~~~~~~ 311 a_4.html 312 a_5.html 313 a_6.html 314 ~~~~~~~~~这是第十八个分隔线~~~~~~~~~