爬取新浪财经个股的历史财报摘要

爬取新浪财经个股的历史财报摘要

网页的内容为:

想要的内容为:

1. pd.DataFrame数据结构
2. 方便查看数据和绘图

print df.tail()
print df.columns
print df.index.name

fig,(ax1,ax2)=plt.subplots(2,1)
# fig.set_figheight(fig.get_figheight()*2)
df.ix[:,(0,3)].plot(ax=ax1)
df.ix[:,5:7].plot(ax=ax2)
ax1.set_ylabel(u'(元)')
ax2.set_ylabel(u'(百万元)')

                      每股净资产    每股收益  每股现金含量  每股资本公积金  固定资产合计   流动资产合计  
南宁糖业(000911)项目：财务摘要                                                     
2015-09-30           4.9220  0.1225 -1.8303   4.4944     NaN  3147.74   
2015-12-31           4.9146  0.1847  0.5290   4.4944     NaN  3032.40   
2016-03-31           4.5619 -0.3527 -3.1519   4.4944     NaN  3868.23   
2016-06-30           4.2956 -0.6190 -3.9426   4.4944     NaN  3522.59   
2016-09-30           4.1173 -0.7973 -2.4654   4.4944     NaN  3194.63   

                        资产总计    长期负债合计    主营业务收入      财务费用       净利润  
南宁糖业(000911)项目：财务摘要                                                   
2015-09-30           4730.00   695.012  2208.240  104.7310   39.7108  
2015-12-31           5669.74   732.949  3138.420  143.6370   59.8534  
2016-03-31           6565.34  1105.460   515.594   39.3776 -114.2920  
2016-06-30           6086.67  1033.750   920.286   90.8180 -200.6000  
2016-09-30           6021.67  1055.570  1624.810  135.9120 -258.3930  
Index([u'每股净资产', u'每股收益', u'每股现金含量', u'每股资本公积金', u'固定资产合计', u'流动资产合计', u'资产总计',
       u'长期负债合计', u'主营业务收入', u'财务费用', u'净利润'],
      dtype='object')
南宁糖业(000911)项目：财务摘要
Out[403]: <matplotlib.text.Text at 0xdde1670>

matplotlib绘图:

代码:

def get_gg_fin_abs(code='000911'):
    u'''
    Note
    -----
    - xpath表达式: 'td[1]' 表示第一个td标签, 其中的'[n]': 声明第几个标签, 1-based
    - etree对象操作: html的上层文字的获取: 
      用  .text属性 比 .xpath('text()')[0]方法 简明得多, 
      前提是: etree.element要具有text属性
    - .find(_path) and .findall(_path) 方法也很好用, 
      他们分别返回一个etree._Element对象, 或者etree._Element对象的list
    - <tbody> tag in <table> is usually added by the browser, not actually 
      in the html source. so you can not .find() or .xpath() it. 
      Use it's parent tag which is <table> to work with.
    Ref
    -----
    - Get all td content inside tr of tbody in python using lxml - Stack Overflow  
    - http://stackoverflow.com/questions/37080910/get-all-td-content-inside-tbody-of-tr-in-python-using-lxml
    '''
    url = 'http://vip.stock.finance.sina.com.cn/corp/go.php/vFD_FinanceSummary/stockid/%s.html'
    url = url%(code)
    craw=crawler.Crawler(url)
    craw.idom()
    
    tr_path='//table[@id="FundHoldSharesTable"]//tr'
    trs = craw.dom.xpath(tr_path)
    print len(trs)
    
    # --------   提取日期行的注意事项            ---------
    # etree.tostring()方法可以查看: html文本
    # print etree.tostring( trs[53].xpath('td')[1])
    # <td align="left" class="tdr"><strong>2015-09-30</strong></td>&#13;
    # 得知: 该文本是加粗的文本: 位于<strong>路径下面:  所以:需要带上/strong后缀
    # k_date = tr_nodes[53].xpath('td[1]/strong').text 
    # v_date = tr_nodes[53].xpath('td[2]/strong').text
    
    #txt =  'text()'
    #txts = 'strong/text()'
    from collections import OrderedDict as Odict
    fdata= Odict()
    #for i,tr in enumerate(trs[53:65+20]):
    # //*[@id="FundHoldSharesTable"]/tbody/tr[1] : 
    # paste this xpath from  copy of Chrome F12 of 审查元素
    # 
    # handle header:  ------------------------------
    #
    header= trs[0].find('th').text
    header=header.strip()
    # ------     handle body data            -------
    for i,tr in enumerate(trs[1:]): # 从第2个tr的数据行开始
        if tr.xpath('td[@height="5px"]') != []: # empty row
            #print 'this is a empty row'
            continue
        if tr.find('td[1]').find('strong') is not  None: # 截止日期行
            vdate = tr.find('td[2]/strong').text  # 找到第二个td的文本
            dict2 = Odict() # 创建一个空的字典, for 本季度的财务摘要数据
            continue
        else:
            k2= tr.find('td[1]').text
#==============================================================================
#             # if tr.find('td[2]/a').text is not None: # 千万不要带属性测试, 因为太贪婪
#             if tr.find('td[2]/a') is not None:
#                 v2= tr.find('td[2]/a').text
#             else:
#                 v2= tr.find('td[2]').text
#==============================================================================
            # 可以更简明地编写为:
            v2= tr.find('td[2]/a').text if tr.find('td[2]/a') is not None else 
                tr.find('td[2]').text 
            dict2[k2]=v2
            # print i,k2,v2 # for debug purpose
            if k2==u'净利润':  # 本季度的最后一行数据, 需要保存小字典到大字典
                fdata[vdate] = dict2
                continue
        
    df=pd.DataFrame(fdata.values(), index=fdata.keys())   
    df.index.name=header
    #print df.head().ix[:,:2]
    #print df #//*[@id="FundHoldSharesTable"]/tbody/tr[870]


    def mapper_strdatetime10_2_datetime(s):
        u'''
        para
        -----
        - s, str, '1998-12-31'
        return
        -----
        - datetime.datetime(1998, 12, 31, 0, 0)
        '''
        y= int(s[0:4])
        m= int(s[5:7])
        d= int(s[8:10])
        return datetime(y,m,d)
    
    def mapper_html_table_td_2_float(td):
        if td.strip() is not u'':
            td = td.strip().replace(u'元','')
            td = float(td)
        else: td = np.nan
        return td

    def mapper_html_table_td_with_comma_2_float(td):
        if td.strip() is not u'':
            td = td.strip().replace(u'元','')
            if td.find(',')>0:
                td = td.replace(',','')
                td = float(td)/1000000.0
                return td
            td = float(td)/1000000.0
            return td
        else: 
            td = np.nan
        return td

    df.index = map(mapper_strdatetime10_2_datetime, df.index)
    df.index.name = header
    
    for i in np.arange(4):  #len(df.columns)):
        df.ix[:,i] = map(mapper_html_table_td_2_float, df.ix[:,i])

    for i in np.arange(4, len(df.columns)):  #len(df.columns)):
        df.ix[:,i] = map(mapper_html_table_td_with_comma_2_float, df.ix[:,i])

    # df.ix[:, :2].plot()
    return df.sort_index()

#==============================================================================
# print df.tail()
# print df.columns
# print df.index.name
# 
# fig,(ax1,ax2)=plt.subplots(2,1)
# # fig.set_figheight(fig.get_figheight()*2)
# df.ix[:,(0,3)].plot(ax=ax1)
# df.ix[:,5:7].plot(ax=ax2)
# ax1.set_ylabel(u'(元)')
# ax2.set_ylabel(u'(百万元)')
#==============================================================================