import json,requests,pymysql
from pprint import pprint
from datetime import datetime
dt=datetime.now()
todayy=datetime(dt.year,dt.month,dt.day,0,0,0)
mysql_110= {
"host": "172.18.28.110",
"port":3306,
"user": "rduser",
"password": "*****8",
"db": "test",
"charset":"utf8"}
#连接mysql
mysql_conn=pymysql.connect(**mysql_110)
sql0="""
CREATE TABLE `jrj_news_original` (
`seq` int(10) unsigned NOT NULL AUTO_INCREMENT COMMENT '序列号，代理主键',
`news_id` int(8) NOT NULL COMMENT '资讯id',
`title` varchar(300) NOT NULL COMMENT '资讯标题',
`detail` mediumtext DEFAULT NULL COMMENT '摘要',
`list_date` datetime NOT NULL COMMENT '发布时间',
`key_word` varchar(100) DEFAULT NULL COMMENT '关键词',
`stk_code` varchar(10) DEFAULT NULL COMMENT '股票代码',
`stk_sname` varchar(30) DEFAULT NULL COMMENT '股票简称',
`img_url` mediumtext DEFAULT NULL COMMENT '图片链接',
`info_url` varchar(100) DEFAULT NULL COMMENT '全文连接',
`content` text DEFAULT NULL COMMENT '内容',
`infocls` varchar(6) DEFAULT NULL COMMENT '栏目ID',
`channum` varchar(3) DEFAULT NULL COMMENT '频道ID',
`cls` int(6) DEFAULT NULL COMMENT '类别ID',
`from_type` int(3) NOT NULL COMMENT '来源类别',
PRIMARY KEY (`seq`),
KEY `list_date` (`list_date`,`news_id`,`stk_code`)
) ENGINE=InnoDB AUTO_INCREMENT=1341 DEFAULT CHARSET=utf8 COMMENT='资讯原表 数据来源：北京提供js格式连接 更新频率：每3分钟更新一次 ';
"""
#创建游标
cursor=mysql_conn.cursor()
#删除当天的数据
deleSql='''
delete from JRJ_NEWS_ORIGINAL
where list_date>=curdate()
'''
cursor.execute(deleSql)
#获取所有查询结果
#datas=cursor.fetchall()
companyUrl='http://stock.jrj.com.cn/share/news/app/company/'+todayy.strftime('%Y-%m-%d')+'.js'
#上市公司 100003 4
hml1=requests.get(companyUrl)
#数据全文
#print(hml1.text[27:][:-6])
tex=json.loads(hml1.text[27:][:-6])
print('上市公司 数据样例： ',tex[0])
len1=len(tex)
print(len1)
for i in range(len1):
if datetime.strptime(tex[i][0]['makedate'],'%Y-%m-%d %H:%M:%S')<todayy:
break
print("----- has insert into( %d ) company datas-----"%(i+1))
#print(tex[i][0]['stockcode'].split(','))
if len(tex[i][0]['stockcode'].split(','))==0:
sql1=''' INSERT INTO JRJ_NEWS_ORIGINAL(NEWs_ID,TITLE,DETAIL,LIST_DATE,IMG_URL,INFO_URL,CLS,FROM_TYPE)
VALUES(%s,%s,%s,%s,%s,%s,100003,4);'''
args1=[tex[i][0]['iiid'],tex[i][0]['title'].replace('"',"'"),tex[i][0]['detail'].replace('"',"'"),tex[i][0]['makedate'],tex[i][0]['imgurl'],tex[i][0]['infourl']]
cursor.execute(sql1,args1)
elif len(tex[i][0]['stockcode'].split(',')) == 1:
sql2='''INSERT INTO JRJ_NEWS_ORIGINAL(NEWs_ID,TITLE,DETAIL,LIST_DATE,STK_CODE,STK_SNAME,IMG_URL,INFO_URL,CLS,FROM_TYPE)
VALUES(%s,%s,%s,%s,%s,%s,%s,%s,100003,4);'''
args2=[tex[i][0]['iiid'],tex[i][0]['title'].replace('"',"'"),tex[i][0]['detail'].replace('"',"'"),tex[i][0]['makedate'],tex[i][0]['stockcode']
,tex[i][0]['stockname'],tex[i][0]['imgurl'],tex[i][0]['infourl']]
cursor.execute(sql2,args2)
else :
for x in range(len(tex[i][0]['stockcode'].split(','))):
sql3='''INSERT INTO JRJ_NEWS_ORIGINAL(NEWs_ID,TITLE,DETAIL,LIST_DATE,STK_CODE,STK_SNAME,IMG_URL,INFO_URL,CLS,FROM_TYPE)
VALUES(%s,%s,%s,%s,%s,%s,%s,%s,100003,4);'''
args3=[tex[i][0]['iiid'],tex[i][0]['title'],tex[i][0]['detail'],tex[i][0]['makedate'],tex[i][0]['stockcode'].split(',')[x]
,tex[i][0]['stockname'].split(',')[x],tex[i][0]['imgurl'],tex[i][0]['infourl']]
cursor.execute(sql3,args3)
#保存提交cursor的执行结果
mysql_conn.commit()
#机会早知道 100001 2
chanceUrl='http://stock.jrj.com.cn/share/news/app/qingbao/'+todayy.strftime('%Y-%m-%d')+'.js'
html2=requests.get(chanceUrl)
#数据全文
#print(hml1.text[26:][:-2])
txt=json.loads(html2.text[26:][:-2])
print('机会早知道 数据样例： ',txt[0])
len1=len(txt)
print(len1)
for i in range(len1):
#print(txt[i]['stockcode'].split(','))
if datetime.strptime(txt[i]['makedate'],'%Y-%m-%d %H:%M:%S')<todayy:
break
print("----- has insert into( %d ) Chance Early Know datas-----"%(i+1))
if len(txt[i]['stockcode'].split(','))==0:
sql1=''' INSERT INTO JRJ_NEWS_ORIGINAL(NEWs_ID,TITLE,DETAIL,LIST_DATE,KEY_WORD,IMG_URL,INFO_URL,CONTENT,CLS,FROM_TYPE)
VALUES(%s,%s,%s,%s,%s,%s,%s,%s,100001,2);'''
args1=[txt[i]['iiid'],txt[i]['title'],txt[i]['detail']
,txt[i]['makedate'],txt[i]['keyword'],txt[i]['imgurl'],txt[i]['infourl'],txt[i]['content']]
cursor.execute(sql1,args1)
#print(''' INSERT INTO NEWS_ORIGINAL(NEWs_ID,TITLE,DETAIL,LIST_DATE,KEY_WORD,IMG_URL,INFO_URL)
#VALUES({0},"{1}","{2}","{3}","{4}","{5}","{6}");'''.format(txt[i]['iiid'],txt[i]['title'],txt[i]['detail']
#,txt[i]['makedate'],txt[i]['keyword'],txt[i]['imgurl'],txt[i]['infourl']
#))
elif len(txt[i]['stockcode'].split(',')) == 1:
sql2=''' INSERT INTO JRJ_NEWS_ORIGINAL(NEWs_ID,TITLE,DETAIL,LIST_DATE,KEY_WORD,STK_CODE,IMG_URL,INFO_URL,CONTENT,CLS,FROM_TYPE)
VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,100001,2);'''
args2=[txt[i]['iiid'],txt[i]['title'],txt[i]['detail'],txt[i]['makedate'],txt[i]['keyword']
,txt[i]['stockcode'],txt[i]['imgurl'],txt[i]['infourl'],txt[i]['content']]
cursor.execute(sql2,args2)
#print('INSERT INTO NEWS_ORIGINAL(NEWs_ID,TITLE,DETAIL,LIST_DATE,KEY_WORD,STK_CODE,IMG_URL,INFO_URL)
#VALUES({0},"{1}","{2}","{3}","{4}","{5}","{6}","{7}");'.
#format(txt[i]['iiid'],txt[i]['title'],txt[i]['detail'],txt[i]['makedate'],default(txt[i]['keyword'])
#,txt[i]['stockcode'],default(txt[i]['imgurl']),default(txt[i]['infourl'])))
else :
for x in range(len(txt[i]['stockcode'].split(','))):
sql3=''' INSERT INTO JRJ_NEWS_ORIGINAL(NEWs_ID,TITLE,DETAIL,LIST_DATE,KEY_WORD,STK_CODE,IMG_URL,INFO_URL,CONTENT,CLS,FROM_TYPE)
VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,100001,2);'''
args3=[txt[i]['iiid'],txt[i]['title'],txt[i]['detail'],txt[i]['makedate'],txt[i]['keyword']
,txt[i]['stockcode'].split(',')[x],txt[i]['imgurl'],txt[i]['infourl'],txt[i]['content']]
cursor.execute(sql3,args3)
#print('INSERT INTO NEWS_ORIGINAL(NEWs_ID,TITLE,DETAIL,LIST_DATE,KEY_WORD,STK_CODE,IMG_URL,INFO_URL)
#VALUES({0},"{1}","{2}","{3}","{4}","{5}","{6}","{7}");'.
#format(txt[i]['iiid'],txt[i]['title'],txt[i]['detail'],txt[i]['makedate'],default(txt[i]['keyword'])
#,txt[i]['stockcode'].split(',')[x],default(txt[i]['imgurl']),default(txt[i]['infourl'])))
#保存提交cursor的执行结果
mysql_conn.commit()
#涨跌停揭秘 100002 3
secretUrl='http://stock.jrj.com.cn/share/news/app/zhangting/'+todayy.strftime('%Y-%m-%d')+'.js'
html3=requests.get(secretUrl)
txt2=json.loads(html3.text[26:][:-2])
len2=len(txt2)
for i in range(len2):
#print(txt2[i]['stockcode'].split(','))
if datetime.strptime(txt2[i]['makedate'],'%Y-%m-%d %H:%M:%S')<todayy:
break
print("----- has insert into( %d ) Secret Up and Down datas-----"%(i+1))
if len(txt2[i]['stockcode'].split(','))==0:
sql1=''' INSERT INTO JRJ_NEWS_ORIGINAL(NEWs_ID,TITLE,DETAIL,LIST_DATE,KEY_WORD,IMG_URL,INFO_URL,CONTENT,CLS,FROM_TYPE)
VALUES(%s,%s,%s,%s,%s,%s,%s,%s,100002,3);'''
args1=[txt2[i]['iiid'],txt2[i]['title'],txt2[i]['detail']
,txt2[i]['makedate'],txt2[i]['keyword'],txt2[i]['imgurl'],txt2[i]['infourl'],txt[i]['content']]
cursor.execute(sql1,args1)
elif len(txt2[i]['stockcode'].split(',')) == 1:
sql2=''' INSERT INTO JRJ_NEWS_ORIGINAL(NEWs_ID,TITLE,DETAIL,LIST_DATE,KEY_WORD,STK_CODE,IMG_URL,INFO_URL,CONTENT,CLS,FROM_TYPE)
VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,100002,3);'''
args2=[txt2[i]['iiid'],txt2[i]['title'],txt2[i]['detail'],txt2[i]['makedate'],txt2[i]['keyword']
,txt2[i]['stockcode'],txt2[i]['imgurl'],txt2[i]['infourl'],txt[i]['content']]
cursor.execute(sql2,args2)
else :
for x in range(len(txt2[i]['stockcode'].split(','))):
sql3=''' INSERT INTO JRJ_NEWS_ORIGINAL(NEWs_ID,TITLE,DETAIL,LIST_DATE,KEY_WORD,STK_CODE,IMG_URL,INFO_URL,CONTENT,CLS,FROM_TYPE)
VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,100002,3);'''
args3=[txt2[i]['iiid'],txt2[i]['title'],txt2[i]['detail'],txt2[i]['makedate'],txt2[i]['keyword']
,txt2[i]['stockcode'].split(',')[x],txt2[i]['imgurl'],txt2[i]['infourl'],txt[i]['content']]
cursor.execute(sql3,args3)
#保存提交cursor的执行结果
mysql_conn.commit()
import random
#乌兰木数据 null 1
noTypeUrl='http://mapp.jrj.com.cn/co/zk/1.js?'+str(random.randint(1,9))+str(random.randint(1,9))
print(noTypeUrl)
hml1=requests.get(noTypeUrl)
tx=hml1.json()
print(len(tx['data']))
len4=len(tx['data'])
data1=tx['data']
for i in range(len4):
print("----- has insert into( %d )datas-----"%i)
if datetime.strptime(data1[i]['makedate'],'%Y-%m-%d %H:%M:%S')<todayy:
break
if len(data1[i]['infostocks'])==0:
sql1=''' INSERT INTO JRJ_NEWS_ORIGINAL(NEWs_ID,TITLE,DETAIL,LIST_DATE,KEY_WORD,IMG_URL,INFO_URL,INFOCLS,CHANNUM,AUTHOR,FROM_TYPE)
VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,1);'''
args1=[data1[i]['iiid'],data1[i]['title'],data1[i]['detail'],data1[i]['makedate'],data1[i]['keyword']
,data1[i]['imgurl'],data1[i]['infourl'],data1[i]['infocls'],data1[i]['channum'],data1[i]['author']]
cursor.execute(sql1,args1)
elif len(data1[i]['infostocks'])== 1:
sql2=''' INSERT INTO JRJ_NEWS_ORIGINAL(NEWs_ID,TITLE,DETAIL,LIST_DATE,KEY_WORD,STK_CODE,STK_SNAME,IMG_URL,INFO_URL,INFOCLS,CHANNUM,AUTHOR,FROM_TYPE)
VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,1);'''
args2=[data1[i]['iiid'],data1[i]['title'],data1[i]['detail'],data1[i]['makedate'],data1[i]['keyword']
,data1[i]['infostocks'][0]['stockcode'],data1[i]['infostocks'][0]['stockname'],data1[i]['imgurl']
,data1[i]['infourl'],data1[i]['infocls'],data1[i]['channum'],data1[i]['author']]
cursor.execute(sql2,args2)
else :
for x in range(len(data1[i]['infostocks'])):
sql3=''' INSERT INTO JRJ_NEWS_ORIGINAL(NEWs_ID,TITLE,DETAIL,LIST_DATE,KEY_WORD,STK_CODE,STK_SNAME,IMG_URL,INFO_URL,INFOCLS,CHANNUM,AUTHOR,FROM_TYPE)
VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,1);'''
args3=[data1[i]['iiid'],data1[i]['title'],data1[i]['detail'],data1[i]['makedate'],data1[i]['keyword']
,data1[i]['infostocks'][x]['stockcode'],data1[i]['infostocks'][x]['stockname'],data1[i]['imgurl']
,data1[i]['infourl'],data1[i]['infocls'],data1[i]['channum'],data1[i]['author']]
cursor.execute(sql3,args3)
#保存提交cursor的执行结果
mysql_conn.commit()
cursor.close()
mysql_conn.close()