python解析网页

果断使用BeautifulSoup！

- -不想写了，挺简单的，贴个代码

import urllib2
import chardet
from BeautifulSoup import BeautifulSoup
import pymongo
import time
db = pymongo.Connection().notice #user notice

def load_work():
    return db.work.find()
def work_insert(data):
    db.work.insert(data)
def update_state(idx,data):
    #print data
    db.state.update(idx,data,upsert=True)
def to_notice(item):
    if(len(item) == 0):
        print 'No news'
    else:
        print item
    #pass
def main_work():
    work = load_work()
    print '=====begin======'
    print time.ctime()
    for i in work:
        data = urllib2.urlopen(i['url']).read()
        charset = chardet.detect(data)['encoding'].lower()
        if charset == 'gb2312':
            charset = 'GBK'
        soup = BeautifulSoup(data , fromEncoding = charset)
        lst = soup.findAll('a')
        new_item = []
        url_set = []
        for item in lst:
            url = item.attrs[0][1]
            text = item.getText()
            url_set.append(url)
            if(db.state.find_one({'url':i['url'] , 'url_set':url}) == None):state.update({'url':url} , {'url':url} , upsert = True)
        if(len(new_item) > 0):
            #TODO notice
            to_notice(new_item)
            db.state.update({'url':i['url']} , {'url':i['url'] , 'url_set':url_set},upsert=True)
    print '=====end======'
while True:
    main_work()
    time.sleep(60)