自学Python十一 Python爬虫总结

　　通过几天的学习与尝试逐渐对python爬虫有了一些小小的心得，我们渐渐发现他们有很多共性，总是要去获取一系列的链接，读取网页代码，获取所需内容然后重复上面的工作，当自己运用的越来越熟练之后我们就会尝试着去总结一下爬虫的共性，试着去写个helper类以避免重复性劳动。

　　参考:用python爬虫抓站的一些技巧总结 zz

　　1.访问网站 #最简单的得到网页代码的方法

import urllib2
response = urllib2.urlopen("http://www.xx.com")
print response.read()

　　2.伪装成浏览器(User-Agent,Referer等) #为了不被服务器禁止访问所以还是伪装成浏览器比较好

headers = {
    'User-Agent': 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)',
    'Referer':'http://www.xx.com/xx',
    'Accept':'application/javascript, */*;q=0.8'
}
response = urllib2.Request(url = "http://www.xx.com",data = None,headers = headers)

　　3.Post数据转码

import urllib,urllib2
values = {
    'username':'xxx',
    'password':'xxx',
    'key':'xxx'
}
postdata = urllib.urlencode(values)
response = urllib2.Request(url,data = postdata)

　　4.Cookies

import urllib2,cookielib
cookie_handler = urllib2.HTTPCookieProcessor(cookielib.CookieJar())
opener = urllib2.build_opener(cookie_handler)
urllib2.install_opener(opener)
response = urllib2.urlopen(url)

　　5.代理服务器 #重复多次访问同一网址 结果被封了ip或限制了访问次数

import urllib2
proxy_handler = urllib2.ProxyHandler({"http" : '42.121.6.80:8080'})
opener = urllib2.build_opener(proxy_handler)
urllib2.install_opener(opener)
response = urllib2.urlopen(url)

　　问：如果想cookie和proxy一起用怎么办？

　　答：urllib2.build_opener可以放多个参数，即handler 如：BaseHandler,ProxyHandler,HTTPHandler,FileHandler,FTPHandler,CacheFTPHandler等等等等

　　6.gzip #现在普遍支持gzip压缩，我们默认获取压缩后的网页，大大提高了抓取网页的效率，减少了带宽负荷。

import urllib2,zlib
req = urllib2.Request(url)
req.add_header('Accept-encoding', 'gzip')
response = urllib2.urlopen(req, timeout=120)
html = response.read()
gzipped = response.headers.get('Content-Encoding')
if gzipped:
    html = zlib.decompress(html, 16+zlib.MAX_WBITS)

　　7.其他

　　设置线程栈大小：栈大小显著影响python的内存占用，方法如下：

from threading import stack_size 2 stack_size(32768*16)

　　设置超时

import socket
socket.setdefaulttimeout(10) #设置10秒后连接超时

　　失败后重试

def get(self,req,retries=3):
    try:
        response = self.opener.open(req)
        data = response.read()
    except Exception , what:
        print what,req
        if retries>0:
            return self.get(req,retries-1)
        else:
            print 'GET Failed',req
            return ''
    return data

　　根据以上内容，我们可以写出便于配置解决重复性工作的自己的helper类：

# -*- coding: utf-8 -*-
import cookielib, urllib, urllib2, socket
import zlib,StringIO
class HttpClient:
  __cookie = cookielib.CookieJar()
  #代理设置，需要时添加（后续设置为多代理切换）
  #__proxy_handler = urllib2.ProxyHandler({"http" : '42.121.6.80:8080'})
  __req = urllib2.build_opener(urllib2.HTTPCookieProcessor(__cookie))#,__proxy_handler)
  __req.addheaders = [
    ('Accept', 'application/javascript, */*;q=0.8'),
    ('User-Agent', 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)')
  ]
  urllib2.install_opener(__req)

  def Get(self, url, refer=None,retries=3):
    try:
      req = urllib2.Request(url)
      req.add_header('Accept-encoding', 'gzip')
      if not (refer is None):
        req.add_header('Referer', refer)
      response = urllib2.urlopen(req, timeout=120)
      html = response.read()
      gzipped = response.headers.get('Content-Encoding')
      if gzipped:
          html = zlib.decompress(html, 16+zlib.MAX_WBITS)
      return html
    except Exception,what:
        print what
        if retries>0:
            return self.Get(url,refer,retries-1)
        else:
            print "Get Failed",url
            return ''
    #except urllib2.HTTPError, e:
    #  return e.read()
    #except socket.timeout, e:
    #  return ''
    #except socket.error, e:
    #  return ''

  def Post(self, url, data, refer=None):
    try:
      req = urllib2.Request(url, urllib.urlencode(data))
      #req = urllib2.Request(url,data)
      if not (refer is None):
        req.add_header('Referer', refer)
      return urllib2.urlopen(req, timeout=120).read()
    except urllib2.HTTPError, e:
      return e.read()
    except socket.timeout, e:
      return ''
    except socket.error, e:
      return ''

  def Download(self, url, file):
    output = open(file, 'wb')
    output.write(urllib2.urlopen(url).read())
    output.close()

  def getCookie(self, key):
    for c in self.__cookie:
      if c.name == key:
        return c.value
    return ''

  def setCookie(self, key, val, domain):
    ck = cookielib.Cookie(version=0, name=key, value=val, port=None, port_specified=False, domain=domain, domain_specified=False, domain_initial_dot=False, path='/', path_specified=True, secure=False, expires=None, discard=True, comment=None, comment_url=None, rest={'HttpOnly': None}, rfc2109=False)
    self.__cookie.set_cookie(ck)

　　至于多线程就参考网上找的这段代码好了，还支持并发。。。

from threading import Thread
from Queue import Queue
from time import sleep
#q是任务队列
#NUM是并发线程总数
#JOBS是有多少任务
q = Queue()
NUM = 2
JOBS = 10
#具体的处理函数，负责处理单个任务
def do_somthing_using(arguments):
    print arguments
#这个是工作进程，负责不断从队列取数据并处理
def working():
    while True:
        arguments = q.get()
        do_somthing_using(arguments)
        sleep(1)
        q.task_done()
#fork NUM个线程等待队列
for i in range(NUM):
    t = Thread(target=working)
    t.setDaemon(True)
    t.start()
#把JOBS排入队列
for i in range(JOBS):
    q.put(i)
#等待所有JOBS完成
q.join()

 　　爬虫就靠一段落吧，更深入的爬虫框架以及html解析库暂时放一放，让我考虑考虑接下来的内容，是pygame还是django！

　　爬虫demo的github地址（刚学着玩git ）：http://git.oschina.net/tabei/Python_spider