高企云爬虫

下面是小編工作中爬取高企云數據的源代碼，增加验证码识别代码：

# -*- coding:utf-8 -*-
import requests
import time,re
import pytesseract
import urllib
import json
from PIL import Image
from selenium import webdriver
from lxml import etree
from connect_monogo import save #Python单利设计模式


class Login(object):

    def __init__(self):
        self.s = requests.session()
        self.driver = webdriver.Chrome()
        #self.driver = webdriver.PhantomJS(executable_path="D:/phantomjs-2.1.1-windows/bin/phantomjs.exe")

    def get_code(self):
        url = 'http://www.hights.cn'
        self.driver.get('http://www.hights.cn/beetl/login/toLogin.html')
        code = self.driver.page_source
        #print(code)
        patten = re.compile('<img id="codeImg" alt="" src="(.*?)" />')
        code_url = patten.findall(code)
        #print code_url
        if len(code_url) ==0:
            return '验证码为空号'
        else:
            #print( url + code_url[0])
            d_url = url + code_url[0]
            print(d_url)
        try:
            urllib.urlretrieve(d_url,'code.jpg')
        except IOError:
            print('验证码链接错误')
        finally:
            im = Image.open('code.jpg')
            hk = pytesseract.image_to_string(im)
            self.driver.find_element_by_name('phone').send_keys('15766264244')
            self.driver.find_element_by_name('password').send_keys('123456789')
            time.sleep(2)
            self.driver.find_element_by_name('code').send_keys(str(hk))
            time.sleep(5)
            self.driver.find_element_by_id('login_btn').click()
            print('登录成功')
            time.sleep(55)


    def Public_list(self):#公司名录,已爬取
        'http://www.hights.cn/beetl/library/list.do?showCount=15&libtype=2&currentPage=2'
        url_01 = 'http://www.hights.cn/beetl/library/list.do?showCount=15&libtype=0&currentPage=1'
        name = requests.get(url_01,headers=headers)
        return(name.text)
        # save.save_monogo({'id':json.loads(name.text,encoding='UTF-8')['data']})

    def Template_model(self):#模板范文
        for page_01 in range(1,3):
            url_02 = 'http://www.hights.cn/beetl/library/list.do?showCount=15&libtype=2&currentPage='+str(page_01)
            s1 = requests.get(url_02,headers=headers)
            return json.loads(s1.text,encoding='UTF-8')['data']
            # save.save_monogo({'id':json.loads(name.text,encoding='UTF-8')['data']})

    def Government_documents(self):#政府文件
        for page_02 in range(1,10):
            url_03 = 'http://www.hights.cn/beetl/library/list.do?showCount=15&libtype=5&currentPage='+str(page_02)
            s2 = requests.get(url_03,headers=headers)
            return json.loads(s2.text,encoding='UTF-8')['data']
            # save.save_monogo({'id':json.loads(name.text,encoding='UTF-8')['data']})

    def Policy_interpretation(self):#政策解读
        for page_03 in range(1,15):
            url_04 = 'http://www.hights.cn/beetl/library/list.do?showCount=15&libtype=4&currentPage='+str(page_03)
            print(url_04)
            s3 = requests.get(url_04,headers=headers)
            return json.loads(s3.text,encoding='UTF-8')['data']
            # save.save_monogo({'id':json.loads(name.text,encoding='UTF-8')['data']})


    def Other_information(self):#其他资料
        for page_04 in range(1,3):
            url_05 ='http://www.hights.cn/beetl/library/list.do?showCount=15&libtype=3&currentPage='+str(page_04)
            s4 = requests.get(url_05,headers=headers)
            return json.loads(s4.text,encoding='UTF-8')['data']
            # save.save_monogo({'id':json.loads(name.text,encoding='UTF-8')['data']})

    def jiexi_data(self):
        #return(save.chaxun())

        pdf_list =[]
        for i_json in self.Other_information():
            #for ij in i_json['id']:
                targid = i_json['id']
                filepath = i_json['pdfpath']
                pdf_url = 'http://www.hights.cn/beetl/mydocument/download?targetId='+str(targid)+'&clazzName=com.fh.entity.system.Library&filePath='+ str(filepath)
                path = targid+filepath
                pdf_list.append(pdf_url)
        print(pdf_list)
        #print(type(str(self.shibie_code())))
        #ss = str(self.shibie_code())
        #self.driver.get('http://www.hights.cn/beetl/login/toLogin.html')
        self.driver.find_element_by_name('phone').send_keys('15766264244')
        self.driver.find_element_by_name('password').send_keys('123456789')
        time.sleep(2)
        self.driver.find_element_by_name('code').send_keys(ss)
        time.sleep(5)
        self.driver.find_element_by_id('login_btn').click()
        time.sleep(4)
        for i in pdf_list:
            self.driver.get(i)
            print('下载完成。。。。')
        time.sleep(400)

def main():
    r = Login()
    r.get_code()
    #r.shibie_code()
    #r.jk()
    #r.jiexi_data()
    #r.login_gaoqiyun()
    #r.Public_list()
    #r.Template_model()
    #print(r.Government_documents())
    #print(r.Policy_interpretation())
    #print(r.jiexi_data())

if __name__ =='__main__':
    main()