爬行百度标题&URL案例

思路：

　　先将需要获取的匹配出，然后可以用"永真"（即while True:）来遍历使得URL可以一直自增变化（百度点击下一页URL的pn参数就增加10）每增加10就爬行一遍URL然后提取一次数据。

#-*-coding:UTF-8-*-

import sys,re,requests,graphics,Tkinter
import easygui as gui

string = raw_input("string is :")
pn = 0
while True:
    url = "http://www.baidu.com/s?wd=%s&pn=%d" % (string, pn)
    pn += 10
    html = requests.get(url).text
    # html = """
    # <div class="c-tools" id="tools_2269957611132062659_2" data-tools='{"title":"织梦CMS 官方网站 - 内容管理系统 - 上海卓卓网络科技有限公司","url":"http://www.baidu.com/link?url=gXtstOFbadX8Lia_Fwwl_AS8VUgXEfqcHe4bpP6Paj-BIGvrYgaUwI4BXvB2M4vg"}'><a class="c-tip-icon"><i class="c-icon c-icon-triangle-down-g"></i></a></div>
    # """
    res = "<div .*? data-tools=(.*?)>.*?</div>"
    con = re.findall(res, html)
    for i in con:
        d = eval(i.strip("'"))#将正则匹配到的json格式的数据转换为字典，eval即为转换。
        print "title:" + d[u'title'] + "  " + d['url']

    num = raw_input(u"e or q:")
    if num == "q":
        exit()

 后期又修改了一下．

#!/usr/bin/env python
#encoding:utf-8
#by i3ekr

import sys,re,requests,time,json
print """
                        
                       #G                    
                       #K                    
                      .Et                    
                      :#                     
                    : ##                     
                    ##Dj K                   
                   .####G###                 
                   E;#####f;                 
                    ########                 
                    #######.                 
                    .i#L#,t                  
                    DEDECMS               
                        
"""
string = raw_input("string is :")
pn = 0
nn = 0
r = requests.session()
head = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Trident/7.0; rv:11.0) like Gecko'}
while True:
    url = "http://www.baidu.com/s?wd=%s&pn=%d" % (string, pn)
    html = r.get(url, headers=head).text
    res = "<div .*? data-tools=(.*?)>.*?</div>"
    con = re.findall(res, html)
    pn += 10
    nn += 1
    try:
      for i in con:
          a = eval(eval(i))
          b = r.get(a.get("url"), headers=head)
          print "[%s] %s"%(nn,b.url)
    except Exception as e:
      pass