爬取易车网所有车系车型数据

下面就是是源代码，我是直接把数据保存在字典并且打印到控制台的

# -*- coding: utf-8 -*-
from lxml import etree
import requests,re,random
import time,redis
from myweb.yichewang import user_agent
import urllib.request
from concurrent.futures import ThreadPoolExecutor

import pymysql
from datetime import datetime
from wxpy import *
#bot = Bot(cache_path=True)#微信端监控运行情况

def run_time(func):
    def wrap(*arg,**kwargs):
        start_time = time.time()
        func(*arg,**kwargs)
        print('获取车型链接类方法运行时间为:',time.time() - start_time)
        return func
        #bot.file_helper.send('获取车型链接类方法运行时间为:',time.time() - start_time)
    return wrap


def try_expect(funcs):
    def getwrap(*arg,**kwargs):
        try:
            funcs(*arg,**kwargs)
            print('时间:',datetime.now(),'-----------未发生异常------------')
            #bot.file_helper.send('时间:',datetime.now(),'未发生异常!!!!!!!!!!!!!!')
        except Exception:
            return '-----------出错啦!!!!!!!!!!'
        #return funcs
    return getwrap

class Yi_car_data(object):

    headers = {'Referer':'http://i.yiche.com/authenservice/login.html?returnurl=http%3A%2F%2Fguangzhou.bitauto.com%2F%3Freferrer%3Dhttp%3A%2F%2Fi.yiche.com%2Fauthenservice%2FAboutPassWord%2FResetPasswordResult.aspx%3Freturnurl%3Dhttp%253a%252f%252fi.yiche.com%252fu27686084%252f'}
    login_url = 'http://i.yiche.com/ajax/Authenservice/login.ashx'
    data ={
        'txt_LoginName':'15766264244',
        'txt_Password':'123456789aa',
        'txt_Code':'',
        'cbx_keepState':'true',
        'returnurl':'http://guangzhou.bitauto.com/?referrer=http://i.yiche.com/authenservice/AboutPassWord/ResetPasswordResult.aspx?returnurl=http://i.yiche.com/u27686084/',
        'guid':'',
        'Gamut':'true'

    }
    cartype_url = 'http://api.car.bitauto.com/CarInfo/getlefttreejson.ashx?tagtype=chexing&pagetype=masterbrand&objid=0'

    def __init__(self):
        #client = redis.Redis(host='127.0.0.1',port=int(6379))
        self.pool = redis.ConnectionPool(host='127.0.0.1', port=6379)
        self.r = redis.Redis(connection_pool=self.pool)
        #self.bot = Bot(cache_path=True)
        self.s_requests = requests.Session()
        html = self.s_requests.post(self.login_url,data=self.data,headers = self.headers,timeout=2)
        print(html.status_code)


    def Car_types_all_datas(self):

        global reponse
        self.headers['User-Agent'] = random.choice(user_agent.user_agent_list)
        try:
            while True:
                time.sleep(1)
                reponse = self.s_requests.get(self.cartype_url,headers=self.headers,timeout=3)
                #print(reponse.text[132:])
                if reponse.status_code ==200 or '奥迪' in reponse.content:
                    break
                else:
                    print('downing field')
        except Exception as e:
            print('------------抛出异常----------------:',e)
        cartypeurl_re_patten = 'url:"(.*?)"'
        cartypename_re_patten = 'name:"(.*?)"'
        patten_cartypeurl = re.compile(cartypeurl_re_patten)
        patten_cartypename = re.compile(cartypename_re_patten)
        cartype_url = patten_cartypeurl.findall(str(reponse.text))
        cartype_name = patten_cartypename.findall(str(reponse.text))
        if len(cartype_url)==0:
            if len(cartype_name)==0:
                print('------------------------解析数据为空---------------------')

        else:
            cartype_datas = list(zip(cartype_url,cartype_name))
            #print(type(str(self.r.get('car_url_name'))))
            try:
                if str(cartype_datas) in str(self.r.get('car_url_name')):
                    print('---------------数据有重复-------------------------')
                else:
                    self.r.set('car_url_name',str(cartype_datas))
            except Exception as e:
                print(e)
                print('----------------------------插入redis失败--------------------')
            #print(list(zip(cartype_url,cartype_name)))
            return cartype_datas

    @try_expect #捕获运行异常
    @run_time #计算方法运行时间
    def car_type_all_info_time_expect(self):
        self.Car_types_all_datas()


    def car_type_all_info(self):
        car_type_url = []
        for url_name in self.Car_types_all_datas():
            #print(url_name)
            car_type_url.append('http://car.bitauto.com/'+str(url_name[0]))
        #print(car_type_url)
        return car_type_url#返回全部车品牌链接数据


    def get_car_info(self,url):
        global req
        self.headers['User-Agent'] = random.choice(user_agent.user_agent_list)
        try:
            req = self.s_requests.get(url,headers=self.headers,timeout=5)
            #print(req.text)
        except:
            print('--------------------出错啦---------------------------!!!!!!!!!!')
        select  = etree.HTML(str(req.text))
        car_url =['http://car.bitauto.com/'+str(i) for i in select.xpath('//li[@class="name"]/a/@href')]
        car_name = select.xpath('//li[@class="name"]/a/@title')

        if len(car_url)==0 or len(car_name)==0:
            print('----------------提取数据失败-----------------------')
        elif len(car_url)!=0 or len(car_name)!=0:
            #print(list(zip(car_url,car_name)))
            return list(zip(car_url, car_name))#返回每款车品牌的车系链接和名字
        else:
            print('#######################################')

    def get_all_cartypes_info(self,infourl):

        data ={}
        global reqs
        self.headers['User-Agent'] = random.choice(user_agent.user_agent_list)
        try:
            time.sleep(1)
            reqs = self.s_requests.get(infourl, headers=self.headers, timeout=5)
            print(req.status_code)
        except Exception as e:
            print('------------出错啦-----------',e)
        carxpath = etree.HTML(str(reqs.text))

        car_type = carxpath.xpath('//a[@class="txt"]/text()')#车型
        car_price = carxpath.xpath('//span[@class="price"]/text()')#车价
        #a =carxpath.xpath('//a[@class="lnk-bzl"]/text()|//a[@class="data"]/text()')
        car_pailiang = '<span class="data" title="(.*?)">(.*?)</span>'#排量
        car_zidong = '<span class="data">(.*?)</span>'#变速箱
        #car_baozhilv = '<a class="lnk-bzl" href="/zhongxingche/baozhilv/" target="_blank" data-channelid="2.21.2032" data_cyslogclickflag="2.21.2032" onclick="BglogPostLog('2.21.2032',this);">(.*?) </a>'#保值率
        #car_youhao = '<a class="data" data-channelid="2.21.855" target="_blank" href="http://car.bitauto.com/quanxinaodia4l/youhao/(.*?)">(.*?) </a>'#油耗
        pailiang = re.compile(car_pailiang).findall(str(reqs.text),re.X)[0][1]
        biansuxiang = re.compile(car_zidong).findall(str(reqs.text))[0]
        car_baozhilv = carxpath.xpath('//a[@class="lnk-bzl"]/text()')[0][:6]
        car_youhao = carxpath.xpath('//a[@class="data"]/text()')[0][:8]
        data['车型'] = car_type
        data['车价'] = car_price
        data['排量'] = pailiang
        data['变速箱'] = biansuxiang
        data['五年保值率'] = car_baozhilv
        data['油耗'] = car_youhao
        print(data)
        #yield data#存储到mongodb


def all_run_main():

    global all_cartype_datas
    count = 0
    tn_car = Yi_car_data()
    # t.Car_types_all_datas()
    tn_car.car_type_all_info_time_expect()
    dn_car = tn_car.car_type_all_info()
    # print(d)
    for j in dn_car:
        count += 1
        #print('正在解析第'+str(j),'
',count)
        try:
            all_cartype_datas = tn_car.get_car_info(j)
        except Exception as e:
            print(e)
        #print(f)
        yield all_cartype_datas


def threading_run_main():
    tn_car = Yi_car_data()
    main = tn_car.get_all_cartypes_info
    car_urls = []
    for jk in list(all_run_main())[0]:
        car_urls.append(jk[0])
    print('链接数量:',len(car_urls))
    pool = ThreadPoolExecutor(100)#----线程数-----
    f = pool.map(main,car_urls)



if __name__ =="__main__":
    threading_run_main()
    import pymongo
    conn = pymongo.MongoClient('localhost', 27017)   #连接服务器
    db = conn.fangtianxia
    items = db.items
    items.insert('')



    '''
    #tn_car = Yi_car_data()
    #tn_car.get_all_cartypes_info('http://car.bitauto.com/aodiq3haiwai/')
    #print(list(all_run_main()))
    #print(list(all_run_main()))
    '''