爬虫下载脚本

download 文件

#!/usr/bin/python
#_*_coding:utf-8 _*_
import  urlparse
import urllib2
import random
import time
from   datetime  import  datetime, timedelta
import socket 
import disk_cache 
DEFAULT_AGENT='WSWP'  # 设置代理
DEFAULT_DELAY=5       #设置下载延迟 为了限制下载速度
DEFAULT_RETRIES=1#发生错误时候尝试的次数
DEFAULT_TIMEOUT=60   
CACHE=disk_cache.DiskCache()
class   Downloader:
   def __init__ (self ,delay=DEFAULT_DELAY,user_agent=DEFAULT_AGENT, proxies=None,num_retries=DEFAULT_RETRIES, timeout=DEFAULT_TIMEOUT, opener=None,cache=CACHE ):
            socket.setdefaulttimeout(timeout)
            self.throttle=Throttle(delay)
            self.user_agent=user_agent
            self.proxies=proxies
            self.num_retries=num_retries
            self.opener=opener
            self.cache=cache


   def __call__(self,url):
       result=None
       print self.cache
       if  self.cache:
            try:
                print  'women doushi'
                result=self.cache[url]
                print result 
                print '123' 
            except  KeyError:
                pass
            else:
                  if self.num_retries>0  and 500< result['code']<600:
                         result=None
       if result is  None:
              self.throttle.wait(url)
              proxy=random.choice(self.proxies)  if self.proxies   else  None
              headers={'User-agent':self.user_agent}
              result=self.download(url,headers, proxy=proxy,num_retries=self.num_retries)
              if self.cache:
                    self.cache[url]=result

       return result['html']

   def  download(self,url, headers,proxy, num_retries, data=None):
            print 'Downloading：', url
            request=urllib2.Request(url,data,headers or {})
            opener=self.opener or  urllib2.build_opener()
            if proxy:
                  proxy_params={urlparse.urlparse(url).scheme:proxy}
                  opener.add_handler(urllib2.ProxyHandler(proxy_params))


            try:
                       

              response=opener.open(request)
              html=response.read()
              code=response.code

            except   Exception   as e :
                   print  'Download error:',  str(e)

                   html=''
                   
                   if num_retries>0  and   500<=code<600:
                           return  self._get(url,headers, prox,num_retries-1,data)


                   else:
                     code=None
            return  {'html':html,'code':code}
                    
class     Throttle:
      def  __init__ (self,delay):
         self.delay=delay
         self.domains={}
      def wait(self,url):
           domain=urlparse.urlsplit(url).netloc
           last_accessed=self.domains.get(domain)
           if self.delay>0 and last_accessed is not None:
                     sleep_secs=self.delay-(datetime.now()-last_accessed).seconds
                     if  sleep_secs >0:
                              time.sleep(sleep_secs)
           self.domains[domain]=datetime.now()
p=Downloader()
x=p('http://www.meituan.com')

缓存disk_cache 脚本

  

import os
import re
import urlparse
import shutil
import zlib
from datetime import datetime, timedelta
try:
    import cPickle as pickle
except ImportError:
    import pickle



class DiskCache:
    """
    Dictionary interface that stores cached 
    values in the file system rather than in memory.
    The file path is formed from an md5 hash of the key.

    >>> cache = DiskCache()
    >>> url = 'http://example.webscraping.com'
    >>> result = {'html': '...'}
    >>> cache[url] = result
    >>> cache[url]['html'] == result['html']
    True
    >>> cache = DiskCache(expires=timedelta())
    >>> cache[url] = result
    >>> cache[url]
    Traceback (most recent call last):
     ...
    KeyError: 'http://example.webscraping.com has expired'
    >>> cache.clear()
    """

    def __init__(self, cache_dir='cache', expires=timedelta(days=30), compress=True):
        """
        cache_dir: the root level folder for the cache
        expires: timedelta of amount of time before a cache entry is considered expired
        compress: whether to compress data in the cache
        """
        self.cache_dir = cache_dir
        self.expires = expires
        self.compress = compress

    
    def __getitem__(self, url):
        """Load data from disk for this URL
        """
        path = self.url_to_path(url)
        if os.path.exists(path):
            with open(path, 'rb') as fp:
                data = fp.read()
                if self.compress:
                    data = zlib.decompress(data)
                result, timestamp = pickle.loads(data)
                if self.has_expired(timestamp):
                    raise KeyError(url + ' has expired')
                return result
        else:
            # URL has not yet been cached
            raise KeyError(url + ' does not exist')


    def __setitem__(self, url, result):
        """Save data to disk for this url
        """
        path = self.url_to_path(url)
        folder = os.path.dirname(path)
        if not os.path.exists(folder):
            os.makedirs(folder)

        data = pickle.dumps((result, datetime.utcnow()))
        if self.compress:
            data = zlib.compress(data)
        with open(path, 'wb') as fp:
            fp.write(data)


    def __delitem__(self, url):
        """Remove the value at this key and any empty parent sub-directories
        """
        path = self._key_path(url)
        try:
            os.remove(path)
            os.removedirs(os.path.dirname(path))
        except OSError:
            pass


    def url_to_path(self, url):
        """Create file system path for this URL
        """
        components = urlparse.urlsplit(url)
        # when empty path set to /index.html
        path = components.path
        if not path:
            path = '/index.html'
        elif path.endswith('/'):
            path += 'index.html'
        filename = components.netloc + path + components.query
        # replace invalid characters
        filename = re.sub('[^/0-9a-zA-Z-.,;_ ]', '_', filename)
        # restrict maximum number of characters
        filename = '/'.join(segment[:255] for segment in filename.split('/'))
        return os.path.join(self.cache_dir, filename)


    def has_expired(self, timestamp):
        """Return whether this timestamp has expired
        """
        return datetime.utcnow() > timestamp + self.expires


    def clear(self):
        """Remove all the cached values
        """
        if os.path.exists(self.cache_dir):
            shutil.rmtree(self.cache_dir)



if __name__ == '__main__':
    cache=DiskCache()
    print cache