test

import requests
from jqdatasdk import *
import tushare as ts
from bs4 import BeautifulSoup
from lxml import etree
from multiprocessing import Pool
import multiprocessing
import time
import datetime
import random
import pandas as pd
import numpy as np
from sqlalchemy import create_engine
import pymysql
import json
import re
import akshare as ak
import warnings
import math

warnings.filterwarnings("ignore")


########################################################### 数据读写操作 ###########################################################
# pandas追加数据
def pandas_data_to_append_handler(data, table):
    engine = create_engine('mysql+pymysql://root:123456@127.0.0.1:3306/quant?charset=utf8')
    data.to_sql(table, con=engine, if_exists='append', index=False)


# pandas替换数据
def pandas_data_to_replace_handler(data, table):
    engine = create_engine('mysql+pymysql://root:123456@127.0.0.1:3306/quant?charset=utf8')
    data.to_sql(table, con=engine, if_exists='replace', index=False)


# mysql删除数据
def mysql_data_delete_handler(sql):
    conn = pymysql.connect("127.0.0.1", "root", "123456", "quant", charset='utf8')
    cursor = conn.cursor()
    cursor.execute(sql)
    conn.commit()
    cursor.close()
    conn.close()

# 数据去重
def mysql_concept_drop_duplicates():
    concept_data = pd.read_sql(sql='''select * from stock_concept''', con=engine)
    print(concept_data)
    concept_data.drop_duplicates(inplace=True)
    mysql_data_delete_handler('''delete from stock_concept''')

    pandas_data_to_append_handler(concept_data, 'stock_concept')


    print(concept_data.head(5))
    print('所属概念数据已去重')

########################################################### Akshare数据 ###########################################################
# 股票账户统计月度
# http://data.eastmoney.com/cjsj/gpkhsj.html
# 单次获取从 201504 开始至今的所有历史数据
def stock_em_account():
    stock_account_df = ak.stock_em_account()

    stock_account_df.rename(
        {'数据日期': 'date', '新增投资者-数量': 'new_investor', '期末投资者-总量': 'end_investor', '沪深总市值': 'total_market_value',
         '沪深户均市值': 'average_market_value'}, axis='columns', inplace=True)

    stock_account_df = stock_account_df[['date', 'new_investor', 'end_investor', 'total_market_value', 'average_market_value']]
    stock_account_df['date'] = stock_account_df['date'].apply(lambda x: (x.replace('年', '-')).replace('月', '-') + '01')
    stock_account_df['total_market_value'] = stock_account_df['total_market_value'] / 10000
    stock_account_df['total_market_value'] = stock_account_df['total_market_value'].apply(lambda x: round(x, 2))

    mysql_data_delete_handler('''delete from stock_account''')
    pandas_data_to_append_handler(stock_account_df, 'stock_account')

    print(stock_account_df.head(5))
    print('月度股票账户统计数据已更新')


# 创新高和新低的股票数量
# https://www.legulegu.com/stockdata/high-low-statistics
# 单次获取指定 market 的所有历史数据
def stock_high_low_count():
    stock_high_low_count_df = ak.stock_a_high_low_statistics(market="all")

    stock_high_low_count_df = stock_high_low_count_df[['date', 'high20', 'low20', 'high60', 'low60', 'high120', 'low120']]

    mysql_data_delete_handler('''delete from stock_high_low_count''')
    pandas_data_to_append_handler(stock_high_low_count_df, 'stock_high_low_count')

    print(stock_high_low_count_df.head(5))
    print('创新高和新低股票统计数据已更新')


# 破净股统计
# https://www.legulegu.com/stockdata/below-net-asset-statistics
# 单次获取所有历史数据
def stock_pjg_count():
    stock_pjg_count_df = ak.stock_a_below_net_asset_statistics()

    mysql_data_delete_handler('''delete from stock_pjg_count''')
    pandas_data_to_append_handler(stock_pjg_count_df, 'stock_pjg_count')

    print(stock_pjg_count_df.head(5))
    print('破净值股票统计数据已更新')


# 外汇和黄金储备
# http://data.eastmoney.com/cjsj/hjwh.html
# 单次返回所有历史数据
def macro_china_fx_gold():
    china_fx_gold_df = ak.macro_china_fx_gold()

    today = time.strftime("%Y%m%d", time.localtime())
    china_fx_gold_df['date'] = china_fx_gold_df['date'].apply(lambda x: (str(today)[:2] + x.replace('年', '-')).replace('月', '-') + '01')

    mysql_data_delete_handler('''delete from china_fx_gold''')
    pandas_data_to_append_handler(china_fx_gold_df, 'china_fx_gold')

    print(china_fx_gold_df.head(5))
    print('中国外汇及黄金储备数据已更新')


# 中国货币供应量
# http://data.eastmoney.com/cjsj/moneysupply.aspx
# 单次返回所有历史数据
def macro_china_money_supply():
    macro_china_money_supply_df = ak.macro_china_money_supply()

    macro_china_money_supply_df.rename({'月份': 'date', 'M2-数量': 'm2', 'M1-数量': 'm1', 'M0-数量': 'm0'}, axis='columns',inplace=True)

    macro_china_money_supply_df = macro_china_money_supply_df[['date', 'm2', 'm1', 'm0']]
    macro_china_money_supply_df['date'] = macro_china_money_supply_df['date'].apply(lambda x: (x.replace('年', '-')).replace('月份', '-') + '01')

    mysql_data_delete_handler('''delete from china_money_supply''')
    pandas_data_to_append_handler(macro_china_money_supply_df, 'china_money_supply')

    print(macro_china_money_supply_df.head(5))
    print('中国货币供应量数据已更新')


# 实时数据更新
# 沪深股通持股排行
# http://data.eastmoney.com/hsgtcg/list.html
# 单次获取指定 market 和 indicator 的所有数据
def stock_hsgt_hold():
    stock_hsgt_hold_df = ak.stock_em_hsgt_hold_stock(market="北向", indicator="今日排行")

    stock_hsgt_hold_df.rename(
        {'HdDate': 'date', 'SCode': 'scode', 'SName': 'sname', 'HYName': 'hyname', 'NewPrice': 'close_price',
         'Zdf': 'pct_chg', 'ShareHold': 'sharehold', 'ShareSZ': 'sharesz', 'ShareHold_Chg_One': 'sharehold_chg_one',
         'ShareSZ_Chg_One': 'sharesz_chg_one'}, axis='columns', inplace=True)

    stock_hsgt_hold_df = stock_hsgt_hold_df[['date', 'scode', 'sname', 'hyname', 'close_price', 'pct_chg', 'sharehold', 'sharesz', 'sharehold_chg_one', 'sharesz_chg_one']]
    stock_hsgt_hold_df['scode'] = stock_hsgt_hold_df['scode'].apply(lambda x: (x + ".SH") if x.startswith('6') == True else (x + ".SZ"))

    pandas_data_to_append_handler(stock_hsgt_hold_df, 'stock_hsgt_hold')

    print(stock_hsgt_hold_df.head(5))
    print('沪深股通个股明细数据已更新')


# 板块行情
# http://finance.sina.com.cn/stock/sl/
# 单次获取指定的板块行情实时数据
def stock_industry_sina():
    label_df = ak.stock_sector_spot(indicator="新浪行业")

    stock_df = pd.DataFrame()
    for label in label_df['label'].tolist():
        industry_detail_df = ak.stock_sector_detail(sector=label)
        industry_detail_df['label'] = label
        stock_df = stock_df.append(industry_detail_df, ignore_index=True)

    stock_industry_detail_df = pd.merge(stock_df, label_df)

    stock_industry_detail_df.rename({'symbol': 'scode', 'name': 'sname', '板块': 'industry'}, axis='columns', inplace=True)

    stock_industry_detail_df = stock_industry_detail_df[['scode', 'sname', 'industry']]
    stock_industry_detail_df['scode'] = stock_industry_detail_df['scode'].apply(lambda x: (x[2:] + ".SH") if x.startswith('sh') == True else (x[2:] + ".SZ"))
    stock_industry_detail_df.drop_duplicates(inplace=True)

    mysql_data_delete_handler('''delete from stock_industry''')
    pandas_data_to_append_handler(stock_industry_detail_df, 'stock_industry')

    print(stock_industry_detail_df.head(5))
    print('所属新浪行业数据已更新')


# 实时数据更新
# 概念行情
# http://finance.sina.com.cn/stock/sl/
# 单次获取指定的概念行情实时数据
def stock_concept_sina():
    label_df = ak.stock_sector_spot(indicator="概念")

    stock_df = pd.DataFrame()
    for label in label_df['label'].tolist():
        concept_detail_df = ak.stock_sector_detail(sector=label)
        concept_detail_df['label'] = label
        stock_df = stock_df.append(concept_detail_df, ignore_index=True)

    stock_concept_detail_df = pd.merge(stock_df, label_df)

    stock_concept_detail_df.rename({'symbol': 'scode', 'name': 'sname', '板块': 'concept'}, axis='columns', inplace=True)

    stock_concept_detail_df = stock_concept_detail_df[['scode', 'sname', 'concept']]
    stock_concept_detail_df['scode'] = stock_concept_detail_df['scode'].apply(lambda x: (x[2:] + ".SH") if x.startswith('sh') == True else (x[2:] + ".SZ"))

    pandas_data_to_append_handler(stock_concept_detail_df, 'stock_concept')

    print(stock_concept_detail_df.head(5))
    print('所属新浪概念数据已更新')


########################################################### Tushare数据 ###########################################################
# 股票列表
def stock_list():
    stock_list_df = pro.stock_basic(list_status='L',fields='ts_code,name,area,market,exchange,list_date,is_hs')

    stock_list_df.rename({'ts_code': 'scode', 'name': 'sname'}, axis='columns', inplace=True)

    stock_list_df['area'] = stock_list_df['area'].apply(lambda x: "广东" if (x == '深圳') == True else x)

    mysql_data_delete_handler('''delete from stock_list''')
    pandas_data_to_append_handler(stock_list_df, 'stock_list')

    print(stock_list_df.head(5))
    print('股票列表数据已更新')


# 日线行情
# 每分钟内最多调取500次，每次5000条数据
def stock_daily():
    stock_daily_df = pro.daily(trade_date=cur_date, fields='ts_code,trade_date,open,high,low,close,pre_close,pct_chg,amount')

    stock_daily_df.rename(
        {'ts_code': 'scode', 'trade_date': 'date', 'open': 'open_price', 'high': 'high_price', 'low': 'low_price',
         'close': 'close_price', 'pre_close': 'pre_close_price'}, axis='columns', inplace=True)

    pandas_data_to_append_handler(stock_daily_df, 'stock_daily')

    print(stock_daily_df.head(5))
    print('股票日线行情数据已更新')


# 指数行情
def index_daily():
    data1 = pro.index_daily(ts_code='000001.SH', trade_date=cur_date, fields='ts_code,trade_date,open,high,low,close,pre_close,pct_chg,amount')
    data2 = pro.index_daily(ts_code='399001.SZ', trade_date=cur_date, fields='ts_code,trade_date,open,high,low,close,pre_close,pct_chg,amount')
    data3 = pro.index_daily(ts_code='399006.SZ', trade_date=cur_date, fields='ts_code,trade_date,open,high,low,close,pre_close,pct_chg,amount')

    index_daily_df = pd.concat([data1, data2, data3], ignore_index=True)

    index_daily_df.rename(
        {'ts_code': 'scode', 'trade_date': 'date', 'open': 'open_price', 'high': 'high_price', 'low': 'low_price',
         'close': 'close_price', 'pre_close': 'pre_close_price'}, axis='columns', inplace=True)

    pandas_data_to_append_handler(index_daily_df, 'index_daily')

    print(index_daily_df.head(5))
    print('指数日线行情数据已更新')


# 每日涨跌停价格
# 单次最多提取4800条记录，可循环调取，总量不限制
def stock_limit_price():
    stock_limit_price_df = pro.stk_limit(trade_date=cur_date, fields='trade_date,ts_code,pre_close,up_limit,down_limit')

    stock_limit_price_df.rename({'trade_date': 'date', 'ts_code': 'scode'}, axis='columns', inplace=True)

    pandas_data_to_replace_handler(stock_limit_price_df, 'stock_limit_price')

    print(stock_limit_price_df.head(5))
    print('股票涨跌停价格数据已更新')


# 每日指标
def stock_daily_basic():
    stock_daily_basic_df = pro.daily_basic(trade_date=cur_date, fields='ts_code,trade_date,turnover_rate_f,volume_ratio,total_share,float_share,free_share')

    stock_daily_basic_df.rename({'ts_code': 'scode', 'trade_date': 'date'}, axis='columns', inplace=True)

    pandas_data_to_append_handler(stock_daily_basic_df, 'stock_daily_basic')

    print(stock_daily_basic_df.head(5))
    print('股票涨跌停价格数据已更新')


# 个股资金流向
def stock_moneyflow():
    stock_moneyflow_df = pro.moneyflow(trade_date=cur_date, fields='ts_code,trade_date,buy_md_amount,sell_md_amount,buy_lg_amount,sell_lg_amount,buy_elg_amount,sell_elg_amount,net_mf_amount')

    stock_moneyflow_df.rename({'ts_code': 'scode', 'trade_date': 'date'}, axis='columns', inplace=True)

    pandas_data_to_append_handler(stock_moneyflow_df, 'stock_moneyflow')

    print(stock_moneyflow_df.head(5))
    print('股票大单资金流向数据已更新')


# 沪深港通资金流向
def hsgt_moneyflow():
    hsgt_moneyflow_df = pro.moneyflow_hsgt(trade_date=cur_date, fields='trade_date,hgt,sgt,north_money')

    hsgt_moneyflow_df.rename({'trade_date': 'date'}, axis='columns', inplace=True)

    pandas_data_to_append_handler(hsgt_moneyflow_df, 'hsgt_moneyflow')

    print(hsgt_moneyflow_df.head(5))
    print('北向资金流向数据已更新')


# 每日涨跌停统计
# 单次最大1000，总量不限制
def stock_limit_list():
    stock_limit_list_df = pro.limit_list(trade_date=cur_date, fields='trade_date,ts_code,close,pct_chg,fc_ratio,fd_amount,first_time,last_time,open_times,strth,limit_type')

    stock_limit_list_df.rename({'trade_date': 'date', 'ts_code': 'scode', 'close': 'close_price'}, axis='columns', inplace=True)

    pandas_data_to_append_handler(stock_limit_list_df, 'stock_limit_list')

    print(stock_limit_list_df.head(5))
    print('涨跌停统计数据已更新')


# shibor利率
def shibor():
    shibor_df = pro.shibor(date=cur_date, fields='date,on,1m')

    shibor_df.rename({'on': 'pre_night_shibor', '1m': 'one_m_shibor'}, axis='columns', inplace=True)

    pandas_data_to_append_handler(shibor_df, 'shibor')

    print(shibor_df.head(5))
    print('shibor利率数据已更新')


# 交易日历
def trade_cal_date():
    t1 = datetime.date.today()
    t2 = t1 - datetime.timedelta(days=250)
    t3 = t2.strftime('%Y%m%d')
    t4 = t1.strftime('%Y%m%d')

    trade_cal_date_df = pro.trade_cal(exchange='SSE', start_date=t3, end_date=t4, is_open=1, fields='exchange,cal_date,is_open,pretrade_date')

    mysql_data_delete_handler('''delete from trade_cal_date''')
    pandas_data_to_append_handler(trade_cal_date_df, 'trade_cal_date')

    print(trade_cal_date_df.head(5))
    print('交易日历数据已更新')


########################################################### jqdatasdk数据 ###########################################################
# 集合竞价
def jq_stock_call_auction():
    stock_df = pro.stock_basic(list_status='L', fields='ts_code,exchange')
    stock_list = stock_df['ts_code'].apply(lambda x: (x[:7] + "XSHG") if x.endswith('SH') == True else (x[:7] + "XSHE")).tolist()

    auth('18829345691', '345691')

    stock_call_auction_df = pd.DataFrame()
    for stock in stock_list:
        stock_call_auction_df = stock_call_auction_df.append(get_call_auction(stock, start_date=add_date, end_date=add_date))

    stock_call_auction_df.rename({'code': 'scode', 'time': 'date', 'current': 'call_price', 'money': 'amount'}, axis='columns', inplace=True)

    stock_call_auction_df = stock_call_auction_df[['scode', 'date', 'call_price', 'amount']]
    stock_call_auction_df['scode'] = stock_call_auction_df['scode'].apply(lambda x: (x[:6] + '.SH') if str(x).endswith('XSHG') == True else (x[:6] + '.SZ'))

    pandas_data_to_append_handler(stock_call_auction_df, 'stock_call_auction')

    print(stock_call_auction_df.head(5))
    print('竞价数据已更新')


########################################################### 东方财富爬虫数据 ###########################################################
# 个股异动数据
# http://quote.eastmoney.com/changes/
def stock_em_change_spider():
    timestamp = int(round(time.time() * 1000))
    url1 = 'http://push2ex.eastmoney.com/getStockCountChanges?type=4,8,16,32,64,128,8193,8194,8201,8202,8203,8204,8207,8208,8209,8210,8211,8212,8213,8214,8215,8216'
    url2 = "http://push2ex.eastmoney.com/getAllStockChanges?type=8201,8202,8193,4,32,64,8207,8209,8211,8213,8215,8204,8203,8194,8,16,128,8208,8210,8212,8214,8216"


    # 获取最大网页数
    max_page_param = {"_": timestamp, "pagesize": '64', "ut": '7eea3edcaed734bea9cbfc24409ed989', "dpt": 'wzchanges'}
    max_page_html = json.loads(
        session.get(url=url1, params=max_page_param, headers={'User-Agent': random.choice(ua_list)}).text)

    # 定义存储
    dict = {}
    stock_change_df = pd.DataFrame()

    # 获取异动数据
    for page in range(0, int(max_page_html['rt'])):
        change_param = {"pageindex": page, "pagesize": '64', "ut": '7eea3edcaed734bea9cbfc24409ed989', "dpt": 'wzchanges'}
        change_html = json.loads(
            session.get(url=url2, params=change_param, headers={'User-Agent': random.choice(ua_list)}).text)

        # 网页数据解析
        if change_html['data'] is None:
            break
        else:
            allstock = change_html['data']['allstock']
            for stock in allstock:
                scode = stock['c']
                sname = stock['n']
                chg_type = stock['t']
                if len(str(stock['tm'])) == 5:
                    chg_time = str(cur_date) + '0' + str(stock['tm'])
                else:
                    chg_time = str(cur_date) + str(stock['tm'])

                if chg_type in ('8201', '8202', '8207', '8209', '8211', '8215', '8204', '8203', '8208', '8210', '8212', '8216'):
                    chg_value = stock['i'] * 100
                    chg_index = '%'
                elif chg_type in ('8193', '8194', '128', '64'):
                    chg_value = stock['i']
                    chg_index = '股'
                else:
                    chg_value = stock['i']
                    chg_index = '元'

                # 数据存储
                dict = {'chg_time': chg_time, 'scode': scode, 'sname': sname, 'chg_type': chg_type, 'chg_value': chg_value, 'chg_index': chg_index}
                stock_change_df = stock_change_df.append(dict, ignore_index=True)

        stock_change_df = stock_change_df[['chg_time', 'scode', 'sname', 'chg_type', 'chg_value', 'chg_index']]
        stock_change_df = stock_change_df[~ (stock_change_df['scode'].str.startswith('1') | stock_change_df['scode'].str.startswith('2') | stock_change_df['scode'].str.startswith('5') | stock_change_df['scode'].str.startswith('9'))]
        stock_change_df['scode'] = stock_change_df['scode'].apply(lambda x: (x + ".SH") if x.startswith('6') == True else (x + ".SZ"))

        pandas_data_to_append_handler(stock_change_df, 'stock_change')

        print(stock_change_df.head(5))
        print('个股异动数据已更新')


# 涨停股池
# http://quote.eastmoney.com/ztb/detail#type=ztgc
def stock_limit_analysis():
    url = 'http://push2ex.eastmoney.com/getTopicZTPool?'

    i = math.floor(random.random() * 10000000 + 1)
    t = int(time.time() * 1000)
    p = 'callbackdata' + str(i + 1) + '('
    q = ');'

    param = {'cb': 'callbackdata' + str(i + 1), 'ut': '7eea3edcaed734bea9cbfc24409ed989', 'dpt': 'wz.ztzt', 'Pageindex': '0', 'pagesize': 920, 'sort': 'fbt:asc', 'date': date, '_': t}

    html = session.get(url=url, params=param, headers={'User-Agent': random.choice(ua_list)}).text
    html = json.loads(html.replace(p, '').replace(q, ''))
    list = html['data']['pool']

    dict = {}
    stock_limit_analysis_df = pd.DataFrame()
    for x in list:
        date = date
        scode = x['c']
        lbc = x['lbc']
        hybk = x['hybk']
        dict = {'date': date, 'code': scode, 'lbc': lbc, 'hybk': hybk}
        stock_limit_analysis_df = stock_limit_analysis_df.append(dict, ignore_index=True)

    stock_limit_analysis_df = stock_limit_analysis_df[['date', 'code', 'lbc', 'hybk']]
    stock_limit_analysis_df['scode'] = stock_limit_analysis_df['scode'].apply(lambda x: (x + ".SH") if x.startswith('6') == True else (x + ".SZ"))

    pandas_data_to_append_handler(stock_limit_analysis_df, 'limit_stock_lbc')

    print(stock_limit_analysis_df.head(5))
    print('涨停股池数据已更新')


########################################################### 开盘啦爬虫数据 ###########################################################
def kpl_hot_concept_spider():
    url = 'https://pclhb.kaipanla.com/w1/api/index.php'

    dict = {}
    code_list = []
    hot_concept_df = pd.DataFrame()

    param = {'c': 'LongHuBang', 'a': 'GetStockList', 'st': 300, 'Time': str(add_date), 'UserID': 399083, 'Token': '71aef0e806e61ad3169ddc9473e37886'}

    # 网页数据抓取
    html = json.loads(session.post(url=url, headers={'User-Agent': random.choice(ua_list)}, data=param).text)['list']

    # 网页数据解析
    for j in html:
        scode = j['ID']
        sname = j['Name']
        code_list.append(scode)

        if len(j['FengKou']) == 0:
            hot_cept = '无'
        else:
            p = j['FengKou'].split(',')
            for i in p:
                hot_cept = i
                dict = {'date': add_date, 'code': scode, 'name': sname, 'hot_cept': hot_cept}
                hot_concept_df = hot_concept_df.append(dict, ignore_index=True)

    hot_concept_df = hot_concept_df[['date', 'scode', 'sname', 'hot_cept']]
    hot_concept_df = hot_concept_df[hot_concept_df['code'].str.startswith('0') | hot_concept_df['code'].str.startswith('3') | hot_concept_df['code'].str.startswith('6')]
    hot_concept_df['scode'] = hot_concept_df['scode'].apply(lambda x: (x + ".SH") if x.startswith('6') == True else (x + ".SZ"))
    hot_concept_df['date'] = hot_concept_df['date'].apply(lambda x: str(x).replace('-', ''))

    pandas_data_to_append_handler(hot_concept_df, 'hot_concept')

    print(hot_concept_df.head(5))
    print('风口概念数据已更新')

    return code_list


def kpl_lhb_spider(code_list):
    url = 'https://pchq.kaipanla.com/w1/api/index.php'
    for code in code_list:
        param = {'c': 'Stock', 'a': 'GetNewOneStockInfo', 'StockID': code, 'Time': add_date, 'UserID': '399083', 'Token': '71aef0e806e61ad3169ddc9473e37886'}

        html = json.loads(session.post(url=url, headers={'User-Agent': random.choice(ua_list)}, data=param).text)['List']

        dict1 = {}
        dict2 = {}
        data1 = pd.DataFrame()
        data2 = pd.DataFrame()
        data = pd.DataFrame()

        for j in html:
            for b in j['BuyList']:
                date = add_date
                scode = code
                direction = 'buy'
                dname = b['Name']
                buy = b['Buy']
                sell = b['Sell']
                try:
                    if b['GroupIcon'][0] == None:
                        groupicon = '空'
                    else:
                        groupicon = b['GroupIcon'][0]
                except:
                    groupicon = '空'

                youziicon = b['YouZiIcon']

                dict1 = {'date': date, 'code': scode, 'direction': direction, 'dname': dname, 'buy': buy, 'sell': sell, 'groupicon': groupicon, 'youziicon': youziicon}
                data1 = data1.append(dict1, ignore_index=True)

            for b in j['SellList']:
                date = date
                scode = code
                direction = 'sell'
                dname = b['Name']
                buy = b['Buy']
                sell = b['Sell']
                try:
                    if b['GroupIcon'][0] == None:
                        groupicon = '空'
                    else:
                        groupicon = b['GroupIcon'][0]
                except:
                    groupicon = '空'

                youziicon = b['YouZiIcon']

                dict2 = {'date': date, 'code': scode, 'direction': direction, 'dname': dname, 'buy': buy, 'sell': sell, 'groupicon': groupicon, 'youziicon': youziicon}
                data2 = data2.append(dict2, ignore_index=True)

        lhb_list_df = pd.concat([data1, data2], axis=0)
        lhb_list_df = lhb_list_df[['date', 'code', 'direction', 'dname', 'buy', 'sell', 'groupicon', 'youziicon']]
        lhb_list_df = lhb_list_df[lhb_list_df['code'].str.startswith('0') | lhb_list_df['code'].str.startswith('3') | lhb_list_df['code'].str.startswith('6')]
        lhb_list_df['code'] = lhb_list_df['code'].apply(lambda x: (x + ".SH") if x.startswith('6') == True else (x + ".SZ"))
        lhb_list_df['date'] = lhb_list_df['date'].apply(lambda x: str(x).replace('-', ''))

        pandas_data_to_append_handler(lhb_list_df, 'lhb_list')

        print(lhb_list_df.head(5))
    print('风口概念数据已更新')


def kpl_stock_concept():
    url = 'https://pchq.kaipanla.com/w1/api/index.php'
    concept_label = []
    label_df = pd.DataFrame()

    # 获取概念ID
    for i in range(0,500,5):
        param1 = {'c': 'PCArrangeData', 'a': 'GetZSIndexPlate', 'SelType': 2, 'ZSType': 5, 'PType': 2, 'POrder': 1,
                       'PStart': '', 'PEnd': '', 'PIndex': i, 'Pst': 15, 'UserID': '399083','Token': '2292739880d01bd81e169e90a1898ebe'}

        html1 = json.loads(session.post(url=url, headers={'User-Agent': random.choice(ua_list)}, data=param1).text)
        if len(html1['plates']['list']) != 0:
            label_df = label_df.append(html1['plates']['list'],ignore_index=True)
        else:
            break

    label_df = label_df.iloc[:,0:2]
    label_df.columns = ['label','concept']

    # 获取概念股明细
    stock_df = pd.DataFrame()
    for label in label_df['label'].tolist():
        for j in range(0,500,5):
            param2 = {'c': 'PCArrangeData', 'a': 'GetZSIndexPlate', 'SelType': 3, 'LType': 6, 'LOrder': 1,
                             'LStart': '', 'LEnd': '', 'LIndex': j, 'Lst': 15, 'PlateID': label, 'UserID': '399083',
                             'Token': '2292739880d01bd81e169e90a1898ebe'}

            html2 = json.loads(session.post(url=url, headers={'User-Agent': random.choice(ua_list)}, data=param2).text)

            if len(html2['stocks']['list']) != 0:
                data = pd.DataFrame(html2['stocks']['list'])
                data['label'] = label
                stock_df = stock_df.append(data, ignore_index=True)
            else:
                break

    stock_df = stock_df.iloc[:, [0, 1, 13]]
    stock_df.columns = ['scode', 'sname','label']

    stock_concept_detail_df = pd.merge(stock_df, label_df)

    stock_concept_detail_df.rename({'symbol': 'scode', 'name': 'sname', '板块': 'concept'}, axis='columns', inplace=True)

    stock_concept_detail_df = stock_concept_detail_df[['scode', 'sname', 'concept']]
    stock_concept_detail_df['scode'] = stock_concept_detail_df['scode'].apply(lambda x: (x + ".SH") if x.startswith('6') == True else (x + ".SZ"))
    stock_concept_detail_df.drop_duplicates(inplace=True)

    pandas_data_to_replace_handler(stock_concept_detail_df, 'stock_concept')

    print(stock_concept_detail_df.head(5))
    print('开盘啦概念数据已更新')


if __name__ == '__main__':
    pro = ts.pro_api('ac16b470869c5d82db5033ae9288f77b282d2b5519507d6d2c72fdd7')
    engine = create_engine('mysql+pymysql://root:123456@127.0.0.1:3306/quant?charset=utf8')
    session = requests.Session()
    ua_list = [
        'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:52.0) Gecko/20100101 Firefox/52.0',
        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36',
        'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.57.2 (KHTML, like Gecko) Version/5.1.7 Safari/534.57.2',
        'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36',
        'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 UBrowser/4.0.3214.0 Safari/537.36',
        'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3314.0 Safari/537.36 SE 2.X MetaSr 1.0'
    ]

    cur_date = time.strftime("%Y%m%d", time.localtime())
    add_date = time.strftime("%Y-%m-%d", time.localtime())

    # stock_em_account()                      # 股票账户统计月度
    # stock_high_low_count()                  # 创新高和新低的股票数量
    # stock_pjg_count()                       # 破净股统计
    # macro_china_fx_gold()                   # 外汇和黄金储备
    # macro_china_money_supply()              # 中国货币供应量
    # stock_hsgt_hold()                       # 沪深股通持股排行
    # stock_industry_sina()                   # 所属板块
    # stock_concept_sina()                    # 所属概念
    kpl_stock_concept()                     # 开盘啦概念
    # mysql_concept_drop_duplicates()         # 所属概念去重

    # stock_list()                            # 股票列表

    # stock_em_change_spider()                # 异动数据
    # stock_limit_analysis()                  # 涨停股池
    # trade_cal_date()                        # 交易日历

    # stock_daily()                           # 日线行情
    # index_daily()                           # 指数行情
    # stock_limit_price()                     # 每日涨跌停价格
    # stock_daily_basic()                     # 每日指标
    # hsgt_moneyflow()                        # 沪深港通资金流向
    # stock_limit_list()                      # 每日涨跌停统计
    # shibor()                                # shibor利率

    # jq_stock_call_auction()                 # 集合竞价
    # code_list = kpl_hot_concept_spider()    # 热门概念，返回code_list
    # kpl_lhb_spider(code_list)               # 龙虎榜明细，传入codelist