爬取智联招聘

创建项目

wljdeMacBook-Pro:PycharmProjects wlj$ scrapy startproject zhilianzhaopin

New Scrapy project 'zhilianzhaopin', using template directory '/usr/local/lib/python3.6/site-packages/scrapy/templates/project', created in:
    /Users/wlj/work/PycharmProjects/zhilianzhaopin

You can start your first spider with:
    cd zhilianzhaopin
    scrapy genspider example example.com


wljdeMacBook-Pro:PycharmProjects wlj$ scrapy genspider --list
Available templates:
  basic
  crawl
  csvfeed
  xmlfeed
wljdeMacBook-Pro:PycharmProjects wlj$ scrapy genspider -t crawl zhaopin sou.zhaopin.com
Created spider 'zhaopin' using template 'crawl' 
wljdeMacBook-Pro:PycharmProjects wlj$ 

 items.py

import scrapy

class ZhilianzhaopinItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    # 职位
    position = scrapy.Field()
    # 公司
    company = scrapy.Field()
    # 职位描述
    position_1 = scrapy.Field()
    # 公司介绍
    company_1 = scrapy.Field()

zhaopin.py

port scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from zhilianzhaopin.items import ZhilianzhaopinItem

class ZhaopinSpider(CrawlSpider):
    name = 'zhaopin'
    allowed_domains = ['sou.zhaopin.com']
    start_urls = ['https://sou.zhaopin.com/jobs/searchresult.ashx?jl=%E5%8C%97%E4%BA%AC&kw=python&p=1']

    rules = (
        Rule(LinkExtractor(allow=r'python&p=d+'),follow = True),
        Rule(LinkExtractor(allow=r'd+.htm'), callback = 'parse_item'),
    )
    def parse_item(self, response):
        i = ZhilianzhaopinItem()
        #i['domain_id'] = response.xpath('//input[@id="sid"]/@value').extract()
        #i['name'] = response.xpath('//div[@id="name"]').extract()
        #i['description'] = response.xpath('//div[@id="description"]').extract()

        # 职位
        i['position'] = response.xpath('//div[@class="fixed-inner-box"]//h1/text()').extract()[0]
        # 公司
        i['company'] = response.xpath('//div[@class="fixed-inner-box"]//a/text()').extract()[0].strip()
        # 职位描述
        i['position_1'] = response.xpath('//div[@class="tab-inner-cont"]/p/text()').extract()
        # 公司介绍
        i['company_1'] = response.xpath('//div[@class="tab-inner-cont"]/div/div/text()').extract()[0].strip()

        yield i

pipelines.py

import json

class ZhilianzhaopinPipeline(object):
    def __init__(self):
        self.filename = open("zhilian.json", "w")

    def process_item(self, item, spider):
        text = json.dumps(dict(item), ensure_ascii = False) + ",
"
        self.filename.write(text)

        return item
    def close_spider(self, spider):
        self.filename.close()

settings.py

BOT_NAME = 'zhilianzhaopin'

SPIDER_MODULES = ['zhilianzhaopin.spiders']
NEWSPIDER_MODULE = 'zhilianzhaopin.spiders'



# Obey robots.txt rules
ROBOTSTXT_OBEY = False

ITEM_PIPELINES = {
    'zhilianzhaopin.pipelines.ZhilianzhaopinPipeline': 300,
}

LOG_FILE = "dg.log"
LOG_LEVEL = "DEBUG"

 执行

scrapy crawl zhaopin