爬虫工具代码

#工具函数，整体测试不行

import requests

import time

'''

def get_html(url):

    # 代理服务器

    print("开始下载url : {}".format(url))

    proxyHost = "http-dyn.abuyun.com"

    proxyPort = "9020"

    # 代理隧道验证信息

    proxyUser = "H58G6G30137G865D"

    proxyPass = "043F1F63DA9899C8"

    proxyMeta = "http://%(user)s:%(pass)s@%(host)s:%(port)s" % {

        "host": proxyHost,

        "port": proxyPort,

        "user": proxyUser,

        "pass": proxyPass,

    }

    proxies = {

        "http": proxyMeta,

        "https": proxyMeta,

    }

    from fake_useragent import UserAgent

    ua = UserAgent()

    print(ua.random)

    headers = {

        "User-Agent": ua.random

    }

    resp = requests.get(url, proxies=proxies, headers=headers)

    resp = resp.content.decode("utf-8")

    return resp

'''

#阿布云代理ip,此处后期改成ip池获取

def get_html_0(url):

    # 代理服务器

    print("开始下载url : {}".format(url))

    proxies = {

        "http:" :"http://117.95.199.208:9999",

        "https" :"https://117.95.199.208:9999"        

    }

    from fake_useragent import UserAgent

    ua = UserAgent()

    print(ua.random)

    headers = {

        "User-Agent": ua.random

    }

    r = requests.get('http://icanhazip.com/',proxies=proxies)

    print(r.text)

    resp = requests.get(url, proxies=proxies, headers=headers)

    return resp

#menu_text = get_html_0(url).content.decode('utf-8')

#time.sleep(random.randint(0,2))

# 用来try测试远程服务器的连接状况

while True:

        try:

            res_text = requests.get(url).text

        except:

            time.sleep(3)

            print('间隔休眠时间，再次处理')

        else:

            break