高度伪装浏览器封装类

1.高度伪装封装类HeadersHelper.py

import urllib.request
import http.cookiejar

class HeadersHelper:
    def __init__(self, url, path=None):
        self.url = urllib.request.quote(url,safe='/:?=', encoding='utf-8')
        self.path = path

    # 设置信息头，高度仿照浏览器
    def set_Headers(self):
        # 添加报头 注意"Accept-Encoding": "gb2312, utf-8" 防止解码而出现乱码
        headers = {"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
                   "Accept-Encoding": "gb2312, utf-8", "Accept-Language": "zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3",
                   "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:52.0) Gecko/20100101 Firefox/52.0",
                   "Connection": "keep-alive", "Host": "baidu.com"
                   }
        cjar = http.cookiejar.CookieJar()
        opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cjar))
        headall = []
        for key, value in headers.items():
            item = (key, value)
            headall.append(item)
        opener.addheaders = headall
        urllib.request.install_opener(opener)

    # 信息返回
    def feedbak_info(self):
        self.set_Headers()
        # 有时候用utf-8,有时候用gbk
        # http://fjrs168.blog.hexun.com 就需要gbk
        try:
            info = urllib.request.urlopen(self.url).read().decode('utf-8')
        except:
            info = urllib.request.urlopen(self.url).read().decode('gbk')
        return str(info)

    # 信息存档
    def save_InFile(self):
        self.set_Headers()
        info = urllib.request.urlopen(self.url).read()
        file = open(self.path, 'wb')
        file.write(info)
        file.close()

2.测试headershelper_test.py

from HeadersHelper import HeadersHelper
url = "https://www.zhibo8.cc"
#==============================
#hh = HeadersHelper(url)
#print(hh.feedbak_info())
#==============================
path = "E:/workspace/PyCharm/codeSpace/books/python_web_crawler_book/chapter6/demo5/headershelper.html"
hh = HeadersHelper(url, path=path)
hh.save_InFile()