爬虫_拉勾网(解析ajax)

拉勾网反爬虫做的比较严，请求头多添加几个参数才能不被网站识别

找到真正的请求网址，返回的是一个json串，解析这个json串即可，而且注意是post传值

通过改变data中pn的值来控制翻页

job_name读取的结果是一个列表     ['JAVA高级工程师、爬虫工程师']   ，而我只想得到里面的字符串，在用job_name[0]的时候，爬取过程中会报下标错误，不知道怎么回事，都看了一遍没问题啊，只能不处理这个列表了

import requests
from lxml import etree
import time
import re

headers = {
    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.75 Safari/537.36',
    'Cookie': '_ga=GA1.2.438907938.1533109215; user_trace_token=20180801154015-21630e48-955e-11e8-ac93-525400f775ce; LGUID=20180801154015-216316d6-955e-11e8-ac93-525400f775ce; WEBTJ-ID=08122018%2C193417-1652dea595ec0a-0ca5deb5d1dcb9-172f1503-2073600-1652dea595f51f; Hm_lvt_4233e74dff0ae5bd0a3d81c6ccf756e6=1533109214,1534073658; LGSID=20180812193418-a61634a5-9e23-11e8-bae9-525400f775ce; PRE_UTM=m_cf_cpt_baidu_pc; PRE_HOST=www.baidu.com; PRE_SITE=https%3A%2F%2Fwww.baidu.com%2Fs%3Fie%3Dutf-8%26f%3D8%26rsv_bp%3D0%26rsv_idx%3D1%26tn%3Dbaidu%26wd%3Dlagou%2520wang%2520%26rsv_pq%3D8f8c8d4500066afd%26rsv_t%3Deaa21Vp5dXD8YMkjoybb0H4UW4n0ReZfxSBiBne14tXfDsN5XKFydx2jVnA%26rqlang%3Dcn%26rsv_enter%3D1%26rsv_sug3%3D12%26rsv_sug1%3D2%26rsv_sug7%3D101%26rsv_sug2%3D0%26inputT%3D1970%26rsv_sug4%3D3064; PRE_LAND=https%3A%2F%2Fwww.lagou.com%2Flp%2Fhtml%2Fcommon.html%3Futm_source%3Dm_cf_cpt_baidu_pc; _gid=GA1.2.849726101.1534073658; X_HTTP_TOKEN=0d3a98743216c83f14e092a5154d2327; LG_LOGIN_USER_ID=5f3d6b41f24b2423bae0a956753632482a2ecabcb36ef97074ff9fdec683ae51; _putrc=3A0548631F844F3C123F89F2B170EADC; JSESSIONID=ABAAABAAAIAACBI302EB6DF4BD54C909C01751F65D4311D; login=true; unick=%E5%88%98%E4%BA%9A%E6%96%8C; showExpriedIndex=1; showExpriedCompanyHome=1; showExpriedMyPublish=1; hasDeliver=0; gate_login_token=c6678499c0aaeb807c1e37249f90d483317ec272762e7060fa18175f49d9c806; index_location_city=%E5%85%A8%E5%9B%BD; TG-TRACK-CODE=search_code; _gat=1; Hm_lpvt_4233e74dff0ae5bd0a3d81c6ccf756e6=1534074635; LGRID=20180812195035-ec87e3ce-9e25-11e8-a37b-5254005c3644; SEARCH_ID=85704a0c1aaf43d4b9f07489311f0707',
    'Referer': 'https://www.lagou.com/jobs/list_%E7%88%AC%E8%99%AB?city=%E5%85%A8%E5%9B%BD&cl=false&fromSearch=true&labelWords=sug&suginput=pachong',
    'Origin': 'https://www.lagou.com'
}

data = {
    'first': 'false',
    'pn': 1,
    'kd': '爬虫'
}

def get_list_page(page_num):
    url = 'https://www.lagou.com/jobs/positionAjax.json?needAddtionalResult=false'
    data['pn'] = page_num
    response = requests.post(url, headers=headers, data=data)
    result = response.json()
    positions = result['content']['positionResult']['result']
    for position in positions:
        positionId = position['positionId']
        position_url = 'https://www.lagou.com/jobs/%s.html' %positionId

        parse_detail_page(position_url)
        # break


def parse_detail_page(url):
    job_information = {}
    response = requests.get(url, headers=headers)
    # response.raise_for_status()
    html = response.text
    html_element = etree.HTML(html)
    job_name = html_element.xpath('//div[@class="job-name"]/@title')
    job_description = html_element.xpath('//dd[@class="job_bt"]//p//text()')
    for index, i in enumerate(job_description):
        job_description[index] = re.sub('xa0', '', i)
    job_address = html_element.xpath('//div[@class="work_addr"]/a/text()')
    job_salary = html_element.xpath('//span[@class="salary"]/text()')
    
    # 字符串处理去掉不必要的信息
    for index, i in enumerate(job_address):
        job_address[index] = re.sub('查看地图', '', i)
    while '' in job_address:
        job_address.remove('')

    # job_address_detail = html_element.xpath('//div[@class="work_addr"]/a[-2]/text()')
    # print(job_address_detail)
    job_information['job_name'] = job_name
    job_information['job_description'] = job_description
    job_information['job_address'] = job_address
    job_information['job_salary'] = job_salary
    print(job_information)


def main():
    for i in range(1, 31):
        get_list_page(i)
        print('='*30 + '第%s页'%i + '='*30)


if __name__ == '__main__':
    main()

运行结果