开启线程池示例
import time
import threading
from concurrent.futures import ThreadPoolExecutor
pool =ThreadPoolExecutor(100)
spider_list = []
#爬虫方法
#page/url 代表爬取第几页或爬取第几个详情url
#def func1(url):
def func1(page):
print("a",page)
pages = 50 #urls=50 # 参数可以是:列表页数/商品列表urls
for page in range(pages):
# 正在运行的线程id
# print('running thread id : %d now=%d' % (threading.get_ident(), url))
print('running thread id : %d now=%d' % (threading.get_ident(), page))
# 将列表页数或商品列表url提交到函数方法
# str_url = pool.submit(func1, url)
str_page = pool.submit(func1,page)
# 完成抓取数据列表舔加到spider
# spider_list.append(str_url)
spider_list.append(str_page)
print("spider_list=",spider_list)
for list in spider_list:
# 完成的结果
list.result()
print('线程全部执行完毕')
一、多线程爬取京东投诉信息
#!/usr/bin/env python
# -*- coding=utf-8 -*-
import json
import threading
import time
from requests_html import HTMLSession
from concurrent.futures import ThreadPoolExecutor
import warnings
warnings.filterwarnings("ignore")
session = HTMLSession()
proxies =None
# 线程池
pool = ThreadPoolExecutor(30)
big_list = []
pool_list = []
def dewu_company(pages):
# 爬取第几页
print("第"+str(pages)+"页")
t=str(int(time.time()*1000))
url = "https://tousu.sina.com.cn/api/company/received_complaints"
headers ={
"authority": "tousu.sina.com.cn",
"method": "GET",
"path": "/api/company/received_complaints?callback=jQuery11120045959640946885205_1584672560291&couid=7046706808&type=1&page_size=10&page=4&_=1584672560295",
"scheme": "https",
"accept": "text/javascript, application/javascript, application/ecmascript, application/x-ecmascript, */*; q=0.01",
"accept-encoding": "gzip, deflate, br",
"accept-language": "zh-CN,zh;q=0.9,en;q=0.8",
# "cookie": "SINAGLOBAL=183.192.8.132_1582353209.520903; UOR=www.baidu.com,tech.sina.com.cn,; __gads=ID=f98d68da093a23f1:T=1582381171:S=ALNI_MaF2_0MnhLJR0sTx2rAnjRyQXnm7w; UM_distinctid=1706d4483c137a-085e50f1dd69ca-37c143e-144000-1706d4483c23fd; lxlrttp=1578733570; U_TRS1=00000017.83e666c5.5e5a9b19.a73e93a1; Apache=58.246.234.18_1584426545.621701; U_TRS2=00000012.37245179.5e706e32.c9bedf3c; TOUSU-SINA-CN=; ULV=1584600854277:2:1:1:58.246.234.18_1584426545.621701:1582381166398; ULOGIN_IMG=tc-6e5332f21698ea872b48ddbbb7971af59a85; CNZZDATA1273941306=1828399964-1584599137-%7C1584669947",
"referer": "https://tousu.sina.com.cn/company/view/?couid=7046706808",
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.90 Safari/537.36",
"x-requested-with": "XMLHttpRequest",
}
params = {
# "callback":"jQuery11120045959640946885205_1584672560291",
"couid":"5650743478", # 1878960481 阿里 5650743478 京东 # 得物 7046706808
"type":"1",
"page_size":"10",
"page":pages,
"_":t,
}
res = session.get(url, params=params, headers=headers, proxies=proxies, verify=False,timeout=5)
# print(res.text)
info_list = res.json()["result"]["data"]["complaints"]
for info in info_list:
# title = info.get("title")
# uid = info.get("uid")
# summary = info.get("summary")
info_url = 'https:' + info['main']['url']
# print(info_url)
return parse_detail(info_url, info)
def parse_detail(info_url,info):
#https://tousu.sina.com.cn/complaint/view/17349163730/
try:
res = session.get(info_url, proxies=proxies, verify=False)
# print(res.text)
new_dict = dict()
new_dict['投诉编号'] = res.html.xpath('//div[@class="ts-d-question"]/ul/li[1]/text()')[0]
new_dict['投诉对象'] = res.html.xpath('//div[@class="ts-d-question"]/ul/li[2]/a//text()')[0]
new_dict['投诉问题'] = res.html.xpath('//div[@class="ts-d-question"]/ul/li[3]/text()')[0]
new_dict['投诉要求'] = res.html.xpath('//div[@class="ts-d-question"]/ul/li[4]/text()')[0]
new_dict['涉诉金额'] = res.html.xpath('//div[@class="ts-d-question"]/ul/li[5]/text()')[0]
new_dict['投诉进度'] = res.html.xpath('//div[@class="ts-d-question"]/ul/li[6]/b/text()')[0]
new_dict['投诉进程详情'] = res.html.xpath('//*[@class="ts-d-steplist"]')[0].text
# 获取投诉图片
img_info_list=[]
img_url = res.html.xpath('//*[@class="example-image-link"]/@href')
for url in img_url:
img_info_list.append("https:"+url)
new_dict['投诉图片'] = img_info_list
# 获取视频列表
vide_id_list = res.html.xpath('//*[@class="video-ico"]/@data-id')
if len(vide_id_list)>=1:
# 投诉视频详情
new_vide_list = []
if vide_id_list:
for vide_id in vide_id_list:
t = int(time.time())
vide_info_url = f'https://api.ivideo.sina.com.cn/public/video/play?video_id={vide_id}&appver=V11220.191231.02&appname=sinaplayer_pc&applt=web&tags=sinaplayer_pc&player=all&jsonp=&plid=2019090301&prid=&uid=&tid=&pid=1&ran=0.34558379845592846&r=https%3A%2F%2Ftousu.sina.com.cn%2Fcomplaint%2Fview%2F17349160365%2F&referrer=&ssid=gusr_pc_{t}&preload=0&uu=60.180.189.200_1581579174.788519&isAuto=1'
# print("vide_info_url=" ,vide_info_url)
res = session.get(vide_info_url, verify=False)
# result = res.encode('utf-8').decode('unicode_escape')
result = json.loads(res.text)
# print("result =",type(result))
new_vide_list.append(result)
if new_vide_list:
new_dict['投诉视频详情'] = new_vide_list
info['投诉详情'] = new_dict
# else:
# new_dict['投诉视频详情'] = None
# info['投诉详情'] = new_dict
big_list.append(new_dict)
print("big_list==",big_list)
except Exception as e:
print(e)
# 写入json 文件
with open('京东投诉信息.json', "a+", encoding = 'utf-8') as fw:
fw.write(json.dumps(big_list,ensure_ascii=False ) + '
')
def main(pages):
startTime = time.time()
# 爬取页数
for page in range(pages):
name = pool.submit(dewu_company,page)
pool_list.append(name)
for n in pool_list:
n.result()
print("全部结束并保存本地")
# 以下写入json文件不能换行,那位大神可以指点下
# with open('京东投诉信息.json', "a+", encoding = 'utf-8') as fw:
# fw.write(json.dumps(big_list,ensure_ascii=False ) + '
')
endTime = time.time()
print('Done, Time cost: %s ' % (endTime - startTime))
if __name__ == '__main__':
# 输入爬取页数
main(20)
20页数据爬取时间：Done, Time cost: 1.6854908466339111
二、多线程爬取阿里详情投诉信息
#!/usr/bin/env python
# -*- coding=utf-8 -*-
import json
import threading
import time
from requests_html import HTMLSession
from concurrent.futures import ThreadPoolExecutor
import warnings
warnings.filterwarnings("ignore")
session = HTMLSession()
proxies =None
def dewu_company():
for page in range(1,20):
print("第"+str(page)+"页")
t=str(int(time.time()*1000))
url = "https://tousu.sina.com.cn/api/company/received_complaints"
headers ={
"authority": "tousu.sina.com.cn",
"method": "GET",
"path": "/api/company/received_complaints?callback=jQuery11120045959640946885205_1584672560291&couid=7046706808&type=1&page_size=10&page=4&_=1584672560295",
"scheme": "https",
"accept": "text/javascript, application/javascript, application/ecmascript, application/x-ecmascript, */*; q=0.01",
"accept-encoding": "gzip, deflate, br",
"accept-language": "zh-CN,zh;q=0.9,en;q=0.8",
# "cookie": "SINAGLOBAL=183.192.8.132_1582353209.520903; UOR=www.baidu.com,tech.sina.com.cn,; __gads=ID=f98d68da093a23f1:T=1582381171:S=ALNI_MaF2_0MnhLJR0sTx2rAnjRyQXnm7w; UM_distinctid=1706d4483c137a-085e50f1dd69ca-37c143e-144000-1706d4483c23fd; lxlrttp=1578733570; U_TRS1=00000017.83e666c5.5e5a9b19.a73e93a1; Apache=58.246.234.18_1584426545.621701; U_TRS2=00000012.37245179.5e706e32.c9bedf3c; TOUSU-SINA-CN=; ULV=1584600854277:2:1:1:58.246.234.18_1584426545.621701:1582381166398; ULOGIN_IMG=tc-6e5332f21698ea872b48ddbbb7971af59a85; CNZZDATA1273941306=1828399964-1584599137-%7C1584669947",
"referer": "https://tousu.sina.com.cn/company/view/?couid=7046706808",
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.90 Safari/537.36",
"x-requested-with": "XMLHttpRequest",
}
params = {
# "callback":"jQuery11120045959640946885205_1584672560291",
"couid":"1878960481", # 1878960481 阿里 5650743478 京东 # 得物 7046706808
"type":"1",
"page_size":"10",
"page":page,
"_":t,
}
res = session.get(url, params=params, headers=headers, proxies=proxies, verify=False,timeout=5)
# print(res.text)
# 开启线程调用商品详情打开
ths = []
info_list = res.json()["result"]["data"]["complaints"]
for info in info_list:
# title = info.get("title")
# uid = info.get("uid")
# summary = info.get("summary")
info_url = 'https:' + info['main']['url']
# print(info_url)
## 开启线程调用商品详情
th = threading.Thread(target=parse_detail,args=(info_url,info))
th.start()
ths.append(th)
if len(ths) > 10:
for th_one in ths:
th_one.join()
ths = []
for th_one in ths:
th_one.join()
def parse_detail(info_url,info):
#https://tousu.sina.com.cn/complaint/view/17349163730/
try:
big_list = []
res = session.get(info_url, proxies=proxies, verify=False)
# print(res.text)
new_dict = dict()
new_dict['投诉编号'] = res.html.xpath('//div[@class="ts-d-question"]/ul/li[1]/text()')[0]
new_dict['投诉对象'] = res.html.xpath('//div[@class="ts-d-question"]/ul/li[2]/a//text()')[0]
new_dict['投诉问题'] = res.html.xpath('//div[@class="ts-d-question"]/ul/li[3]/text()')[0]
new_dict['投诉要求'] = res.html.xpath('//div[@class="ts-d-question"]/ul/li[4]/text()')[0]
new_dict['涉诉金额'] = res.html.xpath('//div[@class="ts-d-question"]/ul/li[5]/text()')[0]
new_dict['投诉进度'] = res.html.xpath('//div[@class="ts-d-question"]/ul/li[6]/b/text()')[0]
new_dict['投诉进程详情'] = res.html.xpath('//*[@class="ts-d-steplist"]')[0].text
# 获取投诉图片
img_info_list=[]
img_url = res.html.xpath('//*[@class="example-image-link"]/@href')
for url in img_url:
img_info_list.append("https:"+url)
new_dict['投诉图片'] = img_info_list
# 获取视频列表
vide_id_list = res.html.xpath('//*[@class="video-ico"]/@data-id')
if len(vide_id_list)>=1:
# print("vide_id_list=",vide_id_list)
# 投诉视频详情
new_vide_list = []
if vide_id_list:
for vide_id in vide_id_list:
t = int(time.time())
vide_info_url = f'https://api.ivideo.sina.com.cn/public/video/play?video_id={vide_id}&appver=V11220.191231.02&appname=sinaplayer_pc&applt=web&tags=sinaplayer_pc&player=all&jsonp=&plid=2019090301&prid=&uid=&tid=&pid=1&ran=0.34558379845592846&r=https%3A%2F%2Ftousu.sina.com.cn%2Fcomplaint%2Fview%2F17349160365%2F&referrer=&ssid=gusr_pc_{t}&preload=0&uu=60.180.189.200_1581579174.788519&isAuto=1'
# print("vide_info_url=" ,vide_info_url)
res = session.get(vide_info_url, verify=False)
# result = res.encode('utf-8').decode('unicode_escape')
result = json.loads(res.text)
# print("result =",type(result))
new_vide_list.append(result)
if new_vide_list:
new_dict['投诉视频详情'] = new_vide_list
info['投诉详情'] = new_dict
# else:
# new_dict['投诉视频详情'] = None
# info['投诉详情'] = new_dict
big_list.append(new_dict)
print("big_list==",big_list,len(big_list))
except Exception as e:
print(e)
with open('阿里投诉信息.json', "a+", encoding = 'utf-8') as fw:
fw.write(json.dumps(big_list,ensure_ascii=False ) + '
')
if __name__ == '__main__':
startTime = time.time()
dewu_company()
endTime = time.time()
print ('Done, Time cost: %s ' % (endTime - startTime))
20页数据爬取时间：Done, Time cost: 20.348562240600586