Python入门-编写抓取网站图片的爬虫-正则表达式

//生命太短 我用Python！

//Python真是让一直用c++的村里孩子长知识了！

这个仅仅是一个测试，成功抓取了某网站1000多张图片。

下一步要做一个大新闻 大工程

#config = utf-8

import urllib
import urllib2
import re
import os

global CNT
CNT = 0

def getHtml(url):
    
#! /usr/bin/env python
    # -*- coding=utf-8 -*- 
    # @Author pythontab.com
    #url="http://pythontab.com"
    req_header = {'User-Agent':'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
    'Accept':'text/html;q=0.9,*/*;q=0.8',
    'Accept-Charset':'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
    'Accept-Encoding':'gzip',
    'Connection':'close',
    'Referer':None #注意如果依然不能抓取的话，这里可以设置抓取网站的host
    }
    req_header_2 = {
    'User-Agent':'Mozilla/5.0 (Windows NT 6.1; rv:28.0) Gecko/20100101 Firefox/28.0'
    }
    
    req_timeout = 5
    #status = urllib.urlopen(url).code
    #print status
    #if status != 200:
    #    print 'Http Error %s' % status
    #    return False
    
    req = urllib2.Request(url,None,req_header_2)
    resp = urllib2.urlopen(req,None,req_timeout)
    html = resp.read()
    return html

def getAllUrl(html):
    reg = r'<a href="(.+)" target='
    theurl = re.compile(reg)
    url = re.findall(theurl,html)
    return url

def getNext(html):
    reg = r"<a href='.+pai/(.+).html'"
    nxtre = re.compile(reg)
    nxt = re.findall(nxtre,html)
    return nxt[0]

def getName(html):
    reg = r'<title>(.+)</title>'
    nare = re.compile(reg)
    name = re.findall(nare,html)
    return name[0]

def getImg(name,html):
    global CNT
    
    reg = r'<img src="(.{0,80}.jpg)" border="0"'
    imgre = re.compile(reg)
    imglist = re.findall(imgre,html)
    
    reg = r'src="(.{0,80}.jpeg)" border'
    imgre = re.compile(reg)
    imglist.extend(re.findall(imgre,html))

    reg = r"<img src='(.{0,80}.jpg)'"
    imgre = re.compile(reg)
    imglist.extend(re.findall(imgre,html))

    reg = r"<img src='(.{0,80}.jepg)'"
    imgre = re.compile(reg)
    imglist.extend(re.findall(imgre,html))

    local = '.\%s-[%sp]' % (name,len(imglist))
    if os.path.exists(unicode(local,'utf-8')):
        return unicode(local,'utf-8')+u'was existed'
    
    os.mkdir(unicode(local,'utf-8'))

    x = 0  
    for imgurl in imglist:
        print imgurl
        urllib.urlretrieve(imgurl,unicode(local+'\%s.jpg' % x,'utf-8'))
        x+=1
        CNT+=1
        
    return unicode('%s: get %s pthoto(s)' % (name,x),'utf-8')



def getAll(num):
    global CNT
    nxt = 164680
    while num > 0:
        
        url = '---%s.html' % nxt
        print nxt
        html = getHtml(url)
        nxt -= 1
        num -= 1
        if html == False:
            print 'Error'
            continue
        
        print getImg(getName(html),html)
    
    return 'done! %s photos!' % str(CNT)

def getAll_update(index):
    global CNT
    num = CNT
    urls = getAllUrl(getHtml(index))
    
    for url in urls:
        html = getHtml('---'+url)
        print getImg(getName(html),html)
    return 'done! %s photos!' % str(CNT-num)


#print getAll(10)
#html = getHtml('---')
#print getNext(html)

x = 3
while x < 50:
    print getAll_update('---' % x)
    x+=1

#print getAll_update('---')

header 伪装成浏览器

正则表达式 http://www.cnblogs.com/huxi/archive/2010/07/04/1771073.html    //我也是刚刚学

基本都是一路百度写出来的