爬取代理IP，并判断是否可用。

# -*- coding:utf-8 -*-
from gevent import monkey
monkey.patch_all()

import urllib2
from gevent.pool import Pool

import requests
import re

class SpiderProxy: 
    def __init__(self):
        self.headers = {
            "Host": "www.xicidaili.com",
            "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.11; rv:47.0) Gecko/20100101 Firefox/47.0",
            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
            "Accept-Language": "en-US,en;q=0.5",
            "Accept-Encoding": "gzip, deflate",
            "Referer": "http://www.xicidaili.com/nn/",
            }
        self.url = 'http://www.xicidaili.com/nn/'
        self.proxy_list = []
        self.re_ip = re.compile(r'(?<![.d])(?:d{1,3}.){3}d{1,3}(?![.d])')
        self.re_port = re.compile(r'<td>(d+)</td>')

    def get_pagesource(self):
        ''' 取得所有1-n页上的代理IP'''
        try:
            num = int(raw_input('please input 1-'))
            for i in range(1, num + 1):
                pageurl = self.url + str(i)
                req = requests.session()
                html = req.get(pageurl, headers=self.headers)
                ip_list = self.re_ip.findall(html.text)
                port_list = self.re_port.findall(html.text)
                proxy_zip = zip(ip_list, port_list)
                for i in proxy_zip:
                    self.proxy_list.append({'http':i[0] + ':' + i[1]})
        except ValueError:
            print 'please input a num!'
        return self.proxy_list

class IsActiveProxyIP:
    def __init__(self):
        self.is_active_proxy_ip = []

    def probe_proxy_ip(self, proxy_ip):
        proxy = urllib2.ProxyHandler(proxy_ip)
        opener = urllib2.build_opener(proxy)
        urllib2.install_opener(opener)
        try:
            html = urllib2.urlopen('http://1212.ip138.com/ic.asp')
            if html:
                self.is_active_proxy_ip.append(proxy_ip)
                return True
            else:
                return False
        except Exception as e:
            return False

if __name__ == '__main__':
    Proxy = SpiderProxy()
    proxy_list = Proxy.get_pagesource()
    proxy_isactive = IsActiveProxyIP()
    pool = Pool(20)
    pool.map(proxy_isactive.probe_proxy_ip, proxy_list)
    with open(r'E:python_demoproxy_ip.txt', 'wb') as f:
        for ip in proxy_isactive.is_active_proxy_ip:
            ip = str(ip)
            f.write(ip[11:-2] + '
')
    print 'file successed written'