无比强大！Python抓取cssmoban站点的模版并下载

Python实现抓取http://www.cssmoban.com/cssthemes站点的模版并下载

实现代码

# -*- coding: utf-8 -*-
import urlparse
import urllib2
import re
import os  
import os.path

URL='http://www.cssmoban.com/cssthemes'

#全局超时设置 
urllib2.socket.setdefaulttimeout(500)

#依据url获取内容
def getUrlContent(url):
    response = urllib2.urlopen(url)
    html = response.read();
    return html

#获取html中的a标签。且格式是<a target="_blank" href="/showcase/*">的
def getAllUrl(html):
    return re.findall('<a[\s]+href="/cssthemes/d+.shtml">.*?
/a>',html)

#获取下载文件的标题
def getDownTitle(html):
    return re.findall('<h1>(.*?
)</h1>',html)

#获取文件下载的url
def getDownUrl(html):
    return re.findall('<a.*?class="button btn-down".*?
/a>',html)

#获取下一页的url
def getNextUrl(html):
    return re.findall('<a.*?
下一页</a>',html)

#下载文件
def download(title,url):
    result = urllib2.urlopen(url).read()
    if os.path.exists("template/")==False:
        os.makedirs("template/")
    newname=("template/"+title.decode('utf-8'))
    newname=newname+'.'+url[url.rfind('.')+1:len(url)]
    open(newname, "wb").write(result)

#记录日志
def i(msg):
    fileobj=open('info.log','a')
    fileobj.write(msg+'
')
    fileobj.close();
    print msg
#记录错误日志
def e(msg):
    fileobj=open('error.log','a')
    fileobj.write(msg+'
')
    fileobj.close();
    print msg
if __name__ == '__main__':

    #print getDownUrl('<a href="http://down.cssmoban.com/cssthemes1/cctp_17_jeans.zip" target="_blank" class="button btn-down" title="免费下载"><i class="icon-down icon-white"></i><i class="icon-white icon-down-transiton"></i>免费下载</a>')
    
    html= getUrlContent(URL)
    i('開始下载：%s' %(URL))
    while True:
        lista= getAllUrl(html);
        #print lista;
        nextPage=getNextUrl(html)
        #print nextPage[0]
        nextUrl=''
        #i('下一页%s'%(nextPage))
        
        if len(nextPage)<=0:
            e('地址：%s。未找到下一页，程序退出' %(nextPage))
            break;
        
        nextUrl=nextPage[0]
        nextUrl=URL+'/'+nextUrl[nextUrl.index('href="')+6:nextUrl.index('" target')]
        #print nextPage
        for a in lista:
            downGotoUrl=''
            try:
                #print a.decode('utf-8')
                downGotoUrl=(URL+''+a[a.index('href="')+6:a.index('">')])
                downGotoUrl=downGotoUrl.replace(URL,'http://www.cssmoban.com')
                #print downGotoUrl
                downHtml=getUrlContent(downGotoUrl)
                #print downHtml
                downTitleList= getDownTitle(downHtml)
                downTitle=''
                if len(downTitleList)>0:
                    downTitle=downTitleList[0]
                #print downTitle
                downUrlList= getDownUrl(downHtml)
                downUrl=''
                if len(downUrlList)>0:
                    downUrl=downUrlList[0]
                downUrl= downUrl[downUrl.index('href="')+6:downUrl.index('" target')]
                #print downUrl
                i('開始下载：%s,文件名称：%s' %(downUrl,downTitle))

                download(downTitle,downUrl)
                i('%s下载完毕。保存文件名称：%s' %(downUrl,downTitle))
            except Exception,e:
                e('地址：%s下载失败，失败信息：' %(downGotoUrl))
                e(str(e))
                

        i('-----------------------------------------')
        i('运行下一页：%s' %(nextUrl))
        html= getUrlContent(nextUrl)