(补充)scrapy框架爬取智联招聘信息--上海python工作

1. job.py:   

  据观察，pageSize参数可以控制页码，一页90个工作,这里做了一个用户交互，输入数字，即下载相应的前几页的工作信息。　

# -*- coding: utf-8 -*-
import scrapy
import json
from zhilianJob.items import ZhilianjobItem


class JobSpider(scrapy.Spider):
    name = 'job'
    # allowed_domains = ['www.sou.zhaopin.com']
    # start_urls可以简写成：https://fe-api.zhaopin.com/c/i/sou?pageSize=90&cityId=538&kw=python&kt=3
    # start_urls = [
    #     'https://fe-api.zhaopin.com/c/i/sou?pageSize=90&cityId=538&salary=0,0&workExperience=-1&education=-1&companyType=-1&employmentType=-1&jobWelfareTag=-1&kw=python&kt=3&=0&_v=0.02964699&x-zp-page-request-id=3e524df5d2b541dcb5ddb82028a5c1b6-1565749700925-710042&x-zp-client-id=2724abb6-fb33-43a0-af2e-f177d8a3e169']
    start_urls=['https://fe-api.zhaopin.com/c/i/sou?pageSize=90&cityId=538&kw=python&kt=3']

    end = int(input('需要下载前几页信息？请输入页码>>>:').strip())

    def parse(self, response):
        # print(response.text)
        # print(111)
        data = json.loads(response.text)
        job = data['data']['results']
        # print(job)
        try:
            for j in job:
                item = ZhilianjobItem()
                item['job_name'] = j['jobName']
                item['job_firm'] = j['company']['name']
                item['job_firmPeople'] = j['company']['size']['name']
                job_url = item['job_url'] = j['positionURL']
                item['job_salary'] = j['salary']
                item['job_type'] = j['jobType']['items'][0]['name']
                item['job_yaoqiu'] = j['eduLevel']['name'] + ',' + j['workingExp']['name']
                item['job_welfare'] = ','.join(j['welfare'])
                yield scrapy.Request(url=job_url,callback=self.detail_parse,meta={'item':item})

                if self.end > 1:
                    for i in range(2, self.end + 1):
                        url = 'https://fe-api.zhaopin.com/c/i/sou?pageSize=%s&cityId=538&kw=python&kt=3' % str(90 * i)
                        # print(111)
                        yield scrapy.Request(url=url, callback=self.parse)
        except Exception as e:
            print(e)

    def detail_parse(self,response):
        # print(response)
        #获取parse() 参数
        # print(2222)
        item = response.meta['item']
        job_address = response.xpath('//div[@class="job-address"]/div/span/text()').extract_first()
        item['job_address'] = job_address
        yield item

2. items.py

　　比之前多了两个字段 job_url,job_address

# -*- coding: utf-8 -*-

# Define here the models for your scraped items
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/items.html

import scrapy


class ZhilianjobItem(scrapy.Item):
    # define the fields for your item here like:
    job_name = scrapy.Field()       #工作名称
    job_firm = scrapy.Field()       #公司名称
    job_firmPeople = scrapy.Field()       #公司人数
    job_url = scrapy.Field()       #公司url
    job_type = scrapy.Field()       #工作类型
    job_salary = scrapy.Field()     #薪水
    job_yaoqiu = scrapy.Field()     #工作要求
    job_welfare = scrapy.Field()    #福利
    job_address = scrapy.Field()    #福利
    pass

3. pipelines.py

　

# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html

import pymysql
class ZhilianjobPipeline(object):
    conn = None
    mycursor = None

    def open_spider(self,spider):
        self.conn = pymysql.connect(host='172.16.25.37',port=3306,user='root',password='root',db='scrapy')
        # 获取游标
        self.mycursor = self.conn.cursor()

        # print('正在清空之前的数据...')
        # # 我只打算要第一页的数据，所以每次爬取都是最新的，要把数据库里的之前的数据要清空
        # sql1 = "truncate table sh_python"
        # self.mycursor.execute(sql1)
        print('上海--python--开始下载...
请稍后...')
    def process_item(self, item, spider):
        # print(0000)
        job_name = item['job_name']
        job_firm = item['job_firm']
        job_firmPeople = item['job_firmPeople']
        job_url = item['job_url']
        job_salary = item['job_salary']
        job_type = item['job_type']
        job_yaoqiu = item['job_yaoqiu']
        job_welfare = item['job_welfare']
        job_address = item['job_address']

        try:
            sql2 = "insert into sh_python VALUES (NULL ,'%s','%s','%s','%s','%s','%s','%s','%s','%s')"%(job_name,job_firm,job_firmPeople,job_salary,job_type,job_yaoqiu,job_welfare,job_url,job_address)
            #执行sql
            self.mycursor.execute(sql2)
            #提交
            self.conn.commit()
        except Exception as e:
            print(e)
            self.conn.rollback()

        return item

    def close_spider(self,spider):
        self.mycursor.close()
        self.conn.close()
        print('上海--python--下载完毕...')

结果: