python 爬取网页内的代理服务器列表（需调整优化）

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Date    : 2017-08-30 20:38:23
# @Author  : EnderZhou (zptxwd@gmail.com)
# @Link    : http://www.cnblogs.com/enderzhou/
# @Version : $Id$

import requests
from bs4 import BeautifulSoup as bs

# 这种爬取网页内容中的列表的方式复用性差，不同的网站需要针对性的修改。每次使用均需要填写更换header头。后续将编写适用性更强的版本。

url = 'http://www.kuaidaili.com/free/inha/'

headers = {
'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
'Accept-Encoding':'gzip, deflate',
'Accept-Language':'zh-CN,zh;q=0.8,zh-TW;q=0.6',
'AlexaToolbar-ALX_NS_PH':'AlexaToolbar/alx-4.0.1',
'Cache-Control':'max-age=0',
'Connection':'keep-alive',
'Cookie':'yd_cookie=a0d0f393-2812-44d0b1453fbf740f3ce870820ada37151e8c; _ydclearance=dd0b3de069ce8a768712e248-d97e-4bd9-8284-f2ef598da35b-1504104455; channelid=0; sid=1504099004948599; _ga=GA1.2.742898386.1504074603; _gid=GA1.2.583101265.1504074603; Hm_lvt_7ed65b1cc4b810e9fd37959c9bb51b31=1504074603,1504097260; Hm_lpvt_7ed65b1cc4b810e9fd37959c9bb51b31=1504099719',
'Host':'www.kuaidaili.com',
'Upgrade-Insecure-Requests':'1',
'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36',
}

def proxy_check(types,ip,port):
    headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36'}
    url = 'http://www.whatismyip.com.tw/'
    proxy = {}
    proxy[types.lower()] = '%s:%s' % (ip,port)
    print proxy
    try:
        r = requests.get(url,headers=headers,proxies=proxy)
        soup = bs(r.content,'html.parser')
        chack_ip = soup.find_all(name='b')
        print chack_ip[0].string+':'+port
    except Exception,e:
        # print e
        pass

# proxy_check('http','183.62.11.242','8088')#可用于测试代理验证模块

def main():
    r = requests.get(url=url,headers=headers)
    tr_soup = bs(r.content,'html.parser')
    tr = tr_soup.find_all(name='tr')
    for i in tr:
        # print i
        td_soup = bs(str(i),'html.parser')
        td = td_soup.find_all(name='td')
        if len(td) != 0:
            ip = str(td[0].string)
            port = str(td[1].string)
            types = str(td[3].string)
            proxy_check(types,ip,port)

if __name__ == '__main__':
    main()