用python做爬虫的例子

主要就是用了两个库，urllib和BeautifulSoup.

作用是从HTML中解析出解梦的查询词和具体的解释。

# -*- coding: utf-8 -*-
import urllib, urllib2
import time, random
from BeautifulSoup import BeautifulSoup

def fetchURL(str_url):

    user_agent = 'Mozilla/5.0 (Windows NT 6.1; WOW64) 
                  AppleWebKit/537.36 (KHTML, like Gecko)'
    values     = {}
    headers    = {'User-Agent': user_agent}
    data       = urllib.urlencode(values)

    content = ''
    
    try:
        request = urllib2.Request(str_url)
        response = urllib2.urlopen(request)
        html = response.read().decode('gb2312')
        content = parse_content_page(html)
    except:
        content = None

    return content

def parse_content_page(html):
    parsed_html = BeautifulSoup(html)
    try:
        title   = parsed_html.body.find('h1', attrs={'class':'art_title'}).text
        content = parsed_html.body.find('div', attrs={'class':'dream_detail'}).text
    except:
        return None
        
    return [title, content]



if __name__ == '__main__':

    foutput = 'jiemeng.txt'
    with open(foutput, 'w') as fout:
        for i in xrange(1, 10):
            reques_url = 'http://tools.2345.com/zhgjm/%s.htm' % str(i)
            x = fetchURL(reques_url)
            if x != None:
                print >>fout, x[0].encode('utf8')[3:-3]
                print >>fout, x[1].encode('utf8')
            
            # sleep for a while between two http requests 
            seconds = random.random()*10 + 2
            time.sleep(seconds)