Python股票信息抓取(三)

最近在看mongodb，然后会用了一些最简单的mongodb的操作，然后想着结合股票信息的数据的抓取，然后将数据存储在mongodb中，对于mongo和数据库的最大的区别是，mongo不需要建表，直接进行存储，然后在选择数据表的时候在进行插入数据的时候要将str格式的字符串转换成json的格式进行插入，这个我在插入数据的时候调试了十多分钟，一直以为是自己字符串的原因，然后看了看插入数据的格式和百度，然后才发现这点。然后我是插入在本机的test.Share表中的，然后其他的注重点就没有什么了～代码写的很丑，冗余也很大，还是会继续更新～并且程序是但进程进行的数据抓取～嗯～ 很蠢～

#-*-coding:utf-8 -*-
import urllib
import re
import json
import urllib2
from lxml import etree
import requests
import time
from Queue import Queue
from pymongo import MongoClient
import matplotlib.pyplot as plt
URL = 'http://quote.fx678.com/exchange/WH'
nation_que = Queue()
client = MongoClient('localhost',27017)
db = client.test
Share = db.Share

def sub_sort(array,array1,low,high):
    key = array[low]
    key1 = array1[low]
    while low < high:
        while low < high and array[high] >= key:
            high -= 1
        while low < high and array[high] < key:
            array[low] = array[high]
            array1[low] = array1[high]
            low += 1
            array[high] = array[low]
            array1[high] = array1[low]
    array[low] = key
    array1[low] = key1
    return low


def quick_sort(array,array1,low,high):
     if low < high:
        key_index = sub_sort(array,array1,low,high)
        quick_sort(array,array1,low,key_index)
        quick_sort(array,array1,key_index+1,high)

def download(url, headers, num_try=2):
    while num_try >0:
        num_try -= 1
        try:
            content = requests.get(url, headers=headers)
            return content.text

        except urllib2.URLError as e:
            print 'Download error', e.reason

    return None

current_quto = Queue()
open_quto = Queue()
high_quto = Queue()
low_quto = Queue()
close_quto = Queue()
update_time = Queue()
def get_type_url():
    headers = {
        'User_agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36',
        'Referer': 'http://quote.fx678.com/exchange/WH',
        'Cookie': 'io=-voMclEjiizK9nWKALqB; UM_distinctid=15f5938ddc72db-089cf9ba58d9e5-31657c00-fa000-15f5938ddc8b24; Hm_lvt_d25bd1db5bca2537d34deae7edca67d3=1509030420; Hm_lpvt_d25bd1db5bca2537d34deae7edca67d3=1509031023',
        'Accept-Language': 'zh-CN,zh;q=0.8',
        'Accept-Encoding': 'gzip, deflate',
        'Accept': '*/*'
    }
    content = download(URL,headers)
    html = etree.HTML(content)
    result = html.xpath('//a[@class="mar_name"]/@href')
    result1 = html.xpath('//td/text()')
    num = 0
    for each in result1:

        if num%6 == 0:
            current_quto.put(each)
            num += 1
        elif num%6 == 1:
            open_quto.put(each)
            num += 1
        elif num%6 == 2:
            high_quto.put(each)
            num += 1
        elif num%6 == 3:
            low_quto.put(each)
            num += 1
        elif num %6 == 4:
            close_quto.put(each)
            num +=1
        elif num %6 == 5:
            update_time.put(each)
            num +=1
    #while not
    for each in result:
        st = each.split('/')
        nation_que.put(st[len(st)-1])

    get_precent()

def get_precent():

    while not nation_que.empty():
        if not update_time.empty():
            time_update = update_time.get(False)
            update_time.task_done()
        if not current_quto.empty():
            new_rates = current_quto.get(False)
            current_quto.task_done()
        if not open_quto.empty():
            opening = open_quto.get(False)
            open_quto.task_done()
        if not high_quto.empty():
            high = high_quto.get(False)
            high_quto.task_done()
        if not low_quto.empty():
            low = low_quto.get(False)
            low_quto.task_done()
        if not close_quto.empty():
            closing = close_quto.get(False)
            close_quto.task_done()

        ss = nation_que.get(False)
        print ss
        print low
        print high
        print time_update
        print new_rates
        print opening

        url = 'http://api.q.fx678.com/history.php?symbol=' + ss +'&limit=288&resolution=5&codeType=8100&st=0.8274405615006541'
        print url
        headers = {'Accept':'application/json, text/javascript, */*; q=0.01',
                'Accept-Encoding':'gzip, deflate',
                'Accept-Language':'zh-CN,zh;q=0.8',
                'Connection':'keep-alive',
                'Host':'api.q.fx678.com',
                'Origin':'http://quote.fx678.com',
                'Referer':'http://quote.fx678.com/symbol/USD',
                'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36'
               }
        num_try = 2
        while num_try >0:
            num_try -= 1
            try:
                content = requests.get(url, headers=headers)
                html = json.loads(content.text)
                st = html['h']
                T_time = html['t']
                if  len(st) > 0 and len(T_time) > 0:
                    draw_pict(ss,T_time,st,time_update,new_rates,opening,high,low,closing)
                break
            except urllib2.URLError as e:
                print 'Download error', e.reason
        nation_que.task_done()
List = []
def draw_pict(name,T_time1,high_rate,time_update,new_rate,opening,high,low,closing):

    High = T_time1
    Time = high_rate
    High_Rate = []
    T_time = []
    mmap = "{"Type":"%s","Current_quto":"%s","Opening_quto":"%s","High_quto":"%s","low_quto":"%s","Closing_quto":"%s","Update_Time":"%s","Real_TIme_infor":{" % (    name, new_rate, opening, high, low, closing, time_update)
    print mmap
    flag = 0
    for each,high1 in zip(T_time1,high_rate):
        if flag == 1:
            mmap += ","
        else:
            flag = 1
        mm = ""%s":"%s""%(each,high1)


        st = time.localtime(float(each))
        mmap += mm
        if st.tm_min == 0:
            T_time.append(st.tm_hour)
            High_Rate.append(high1)
        else:
            pass
    mmap += "}}"
    mmap1 = json.loads(mmap)
    print mmap1
    Share.insert(mmap1)
    if len(T_time) == len(High_Rate):
        quick_sort(T_time,High_Rate,0,len(High_Rate)-1)
        List.append(High_Rate)

def draw_picture():
    colu = len(List)

    num = 1
    for each in List:
        plt.subplot(colu/2 + 1,2,num)
        num+=1

        list = each
        T_time = []
        for i in range(len(list)):
            T_time.append(i)
        print len(list)
        print len(T_time)
        plt.plot(T_time, list, marker='*')

    plt.show()
    plt.title('Share Message')

if __name__ == '__main__':
    get_type_url()
    draw_picture()