去哪儿

'''
爬取去哪儿网所有城市自由行数据
爬取速度过快时会引发服务器返回错误
反反爬策略:设置cookies池和ip代理池以及延长爬虫休眠时间
'''
import requests
import time
from urllib.parse import quote
from multiprocessing import Pool
import pymongo
import random

def begin():
    """
    获取去哪儿网出发地站点列表
    :return:
    """
    depurl='https://touch.dujia.qunar.com/depCities.qunar'
    response=requests.get(depurl)
    deps=response.json()
    for dep_item in deps['data']:
        for dep in deps['data'][dep_item]:
            yield dep#出发城市

def main(dep):
    """
    获取去哪儿网出发地可旅行的目的地列表
    :param dep: 出发地
    :return: 目的地列表
    """
    a = []
    #中文字符要进行编码
    desurl = 'https://touch.dujia.qunar.com/golfz/sight/arriveRecommend?dep={}&exclude=&extensionImg=255,175'.format(
        quote(dep))
    time.sleep(random.randint(2,4))
    response = requests.get(desurl)
    des = response.json()
    for des_item in des['data']:
        for des_item_1 in des_item['subModules']:
            for query in des_item_1['items']:
                if query['query'] not in a:#去重,目的城市中有重复出现
                    a.append(query['query'])#目的城市列表
    get(a, dep)

def get(array,dep):
    """
    得到去哪儿网自由行数据搜索结果
    :param array: 目的城市列表
    :param dep: 出发城市
    :return:出发城市到目的城市的自由行结果
    """
    for item in array:
        # 头文件 防止反爬
        headers = {
            'cookie':'QN99=8770; QN1=eIQjmVtYQgbBDaEiPevvAg==; csrfToken=zKMVroGqYK6fdBphXg8rqQ3MpcaiZ7TZ; QN269=AA9586A58FEC11E88A24FA163E233FC1; QN601=3f55b4673bbd18ac3206bfea7c5996d3; QunarGlobal=10.86.213.148_6291bf49_164d0ba9dbf_-1a4d|1532510727219; _i=RBTKSaIAM3KBlurx6OwRjfuQ8pEx; QN300=auto_4e0d874a; QN163=0; QN6=auto_4e0d874a; QN48=tc_427b9f2555dccb4c_164d9787381_d960; _RSG=Ue4lzWGVuXAKnGpozKI.OB; _RDG=28c738c8ddc979203b2642a9f86b2ac273; _RGUID=a8787d08-3dbc-4a1e-b63e-494f72cd0c54; QN205=auto_4e0d874a; QN234=home_free_t; _vi=Xan8_FldA2NGBwqzRSKDNIYHisxd4ARxiomsg1mowQsC4OV3wCXnooJECkbZWsL9_3XGq9mmj5lTyMlGPRfgZD0jC_eS-Vas8fJyOdtOVO02USpBUqqwRZ1LfhiofVGvkPVi9NW0omogB1BkpWCaX2atkxba7uWItHjFuSd5R2NK; QN162=%E6%B7%B1%E5%9C%B3; QN233=FreetripTouchin; DJ12=eyJxIjoi5p2t5bee6Ieq55Sx6KGMIiwic3UiOiI4MDU5MjU4OTIiLCJkIjoi5rex5ZyzIiwiZSI6IkEiLCJsIjoiMCwyOCIsInRzIjoiZGQxNDZmZWYtMWY2NC00N2U5LWIyNjAtMTY0ODE2ZTlmYmQ0In0; _RF1=113.110.176.137; _pk_ref.1.8600=%5B%22%22%2C%22%22%2C1533395038%2C%22http%3A%2F%2Ftouch.qunar.com%2F%22%5D; _pk_ses.1.8600=*; _pk_id.1.8600=92302397325aca81.1533353790.5.1533395068.1533392908.; QN243=168',
            'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 '
                          '(KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36',
            'Referer': 'https://touch.dujia.qunar.com/p/list?cfrom=zyx&dep={}&query={}&it=FreetripTouchin&et=home_free_t'.format(quote(dep),quote(item))
        }
        resulturl = 'https://touch.dujia.qunar.com/list?modules=list%2CbookingInfo%2CactivityDetail&dep={}&query={}&dappDealTrace=false&mobFunction=%E6%89%A9%E5%B1%95%E8%87%AA%E7%94%B1%E8%A1%8C&cfrom=zyx&it=FreetripTouchin&date=&configDepNew=&needNoResult=true&originalquery={}&limit=0,28&includeAD=true&qsact=search'.format(
            quote(dep), quote(item), quote(item))
        time.sleep(random.randint(2,4))
        response = requests.get(resulturl, headers=headers).json()

        #容错处理,防止json文件中有不存在的项引起报错
        try:
            routecount = int(response['data']['limit']['routeCount'])#获取
            for limit in range(0, routecount, 28):
                resulturl = 'https://touch.dujia.qunar.com/list?modules=list%2CbookingInfo%2CactivityDetail&dep={}&query={}' 
                            '&dappDealTrace=false&mobFunction=%E6%89%A9%E5%B1%95%E8%87%AA%E7%94%B1%E8%A1%8C&cfrom=zyx&' 
                            'it=FreetripTouchin&date=&configDepNew=&needNoResult=true&originalquery={}&limit={},28&' 
                            'includeAD=true&qsact=search'.format(quote(dep), quote(item), quote(item), limit)
                time.sleep(random.randint(2,4))
                response = requests.get(resulturl, headers=headers)
                items=response.json()['data']['list']['results'][0]
                result = {
                    '时间': time.strftime('%Y-%m-%d', time.localtime(time.time())),
                    '出发地': dep,
                    '目的地': item,
                    '价格':items['price'],
                    '天数': items['accomInclude'],
                    '亮点': items['brightspots'],
                    '出行工具':items['backtraffic'],
                    '类别':items['ttsRouteType']
                }
                print(result)
                savetomongo(result)
                time.sleep(1)
        except:
            return

client=pymongo.MongoClient(host='127.0.0.1',port=27017)#连接mongodb
db=client['qunar']#mongodb数据库
collection='travel'#mongodb集合


def savetomongo(result):
    """
    保存到mongodb数据库
    :param result: 出发城市到目的城市自由行搜索结果
    :return:
    """
    db[collection].insert(result)#插入数据到mongodb


if __name__ == '__main__':
    deps = begin()
    #开启多线程
    pool=Pool()
    pool.map(main,[dep for dep in deps])
    client.close()