一个爬取股票信息的爬虫程序

　　给大家分享一个早前爬取东方财富网股票信息的爬虫程序，回头来看做了好多改进，特别是数据处理部分使用了heapd模块，方便快捷一步到位...

# _*_ coding:utf-8 _*_

import requests,re,json,time,os
import heapq
from bs4 import BeautifulSoup

class GPINFO(object):
    """docstring for GPINFO"""
    def __init__(self):
        self.Url = 'http://quote.eastmoney.com/stocklist.html'
        self.BaseData = []
        self.Date = time.strftime('%Y%m%d')
        self.Record = 'basedata'+self.Date
        if os.path.exists(self.Record):
            print ('record exist...')
            self.BaseData = self.get_base_data_from_record()
        else:
            print ('fuck-get data again...')
            self.get_data()

    def write_record(self,text):
        with open(self.Record,'ab') as f:
            f.write((text+'
').encode('utf-8'))

    def get_base_data_from_record(self):
        ll = []
        with open(self.Record,'rb') as f:
            json_l = f.readlines()
            for j in json_l:
                ll.append(json.loads(j.decode('utf-8')))
        return ll

    def get_data(self):
        #请求数据
        orihtml = requests.get(self.Url).content
        #创建 beautifulsoup 对象
        soup = BeautifulSoup(orihtml,'lxml')
        #采集每一个股票的信息
        count = 0
        for a in soup.find('div',class_='quotebody').find_all('a',{'target':'_blank'}):
            record_d = {}
            #代号
            num = a.get_text().split('(')[1].strip(')')
            if not (num.startswith('00') or num.startswith('60')):continue #只需要6*/0*
            record_d['num']=num
            #名称
            name = a.get_text().split('(')[0]
            record_d['name']=name
            #详情页
            detail_url = a['href']
            record_d['detail_url']=detail_url

            cwzburl = detail_url
            #发送请求
            try:
                cwzbhtml = requests.get(cwzburl,timeout=30).content
            except Exception as e:
                print ('perhaps timeout:',e)
                continue
            #创建soup对象
            cwzbsoup = BeautifulSoup(cwzbhtml,'lxml')

            #财务指标列表 [浦发银行，总市值    净资产    净利润    市盈率    市净率    毛利率    净利率    ROE] roe:净资产收益率
            try:
                cwzb_list = cwzbsoup.find('div',class_='cwzb').tbody.tr.get_text().split()
            except Exception as e:
                print ('error:',e)
                continue
            #去除退市股票
            if '-' not in cwzb_list:
                record_d['data']=cwzb_list
                self.BaseData.append(record_d)
                self.write_record(json.dumps(record_d))
                count=count+1
                print (len(self.BaseData))

def main():
    test = GPINFO()
    result = test.BaseData
    #[浦发银行，总市值    净资产    净利润    市盈率    市净率    毛利率    净利率    ROE] roe:净资产收益率]
    top_10 = heapq.nlargest(10,result,key=lambda r:float(r['data'][7].strip('%')))
    for i in top_10:
        print(i['data'])

if __name__ == '__main__':
    main()

　　程序主函数部分是为了获取净利率前10名的股票信息,打印结果如下:

['绵石投资', '52.2亿', '14.0亿', '1.25亿', '30.90', '3.73', '42.25%', '2047.04%', '9.27%']
['国投安信', '556亿', '270亿', '21.1亿', '19.80', '2.12', '5.90%', '487.53%', '7.79%']
['川投能源', '379亿', '202亿', '28.0亿', '10.16', '1.91', '37.01%', '402.64%', '14.58%']
['ST明科', '47.6亿', '9.25亿', '5.11千万', '68.00', '5.14', '2.38%', '345.11%', '5.68%']
['华联控股', '93.6亿', '31.5亿', '4.76亿', '14.54', '3.74', '46.25%', '328.53%', '20.88%']
['上海九百', '68.2亿', '12.3亿', '1.61亿', '31.67', '5.56', '54.00%', '297.99%', '13.21%']
['凯瑞德', '46.7亿', '1.14亿', '3.27千万', '107.10', '40.94', '16.07%', '294.19%', '33.41%']
['鲁信创投', '172亿', '38.6亿', '3.32亿', '38.48', '4.64', '28.67%', '244.43%', '9.26%']
['博闻科技', '35.0亿', '6.56亿', '2.23千万', '117.65', '5.36', '-16.07%', '215.27%', '3.41%']
['万泽股份', '71.8亿', '13.7亿', '6.87千万', '78.38', '5.29', '22.57%', '203.15%', '5.13%']