初识Scrapy

Scrapy常规命令与文件目录解读

Scrapy
    # 创建project
    scrapy startproject wyb
    wyb
        wyb/
            spiders           # 爬虫文件
                chouti.py
            items.py          # 持久化
            middlewares.py    # 中间件
            pipelines.py      # 持久化
            settings.py       # 项目配置文件
        scrapy.cfg            # 配置文件（部署）
      
      
    cd wyb
    # 创建爬虫
    scrapy genspider chouti chouti.com
    
    # 启动爬虫
    scrapy crawl chouti
    scrapy crawl chouti --nolog  # 不打印日志

简单实例：用scrapy爬取抽屉热榜

# -*- coding: utf-8 -*-
import sys
import os
import io
import scrapy

# sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='gb18030') 编码问题


class ChoutiSpider(scrapy.Spider):
    name = 'chouti'
    # 爬取定向的网页 只允许这个域名的
    allowed_domains = ['chouti.com']
    start_urls = ['http://chouti.com/']

    def parse(self, response):
        from scrapy.http.response.html import HtmlResponse
        # print(response, type(response))
        # print(response.text)
        f = open('new.log', 'a+')
        item_list = response.xpath('//div[@id="content-list"]/div[@class="item"]')
        for item in item_list:
            text = item.xpath('.//a/text()').extract_first()
            href = item.xpath('.//a/@href').extract_first()
            print(href, text.strip())
            f.write(href+'
')
        f.close()

        page_list = response.xpath('//div[@id="dig_lcpage"]//a/@href').extract()
        for page in page_list:
            from scrapy.http import Request
            page = "https://dig.chouti.com"+page
            # 继续发请求，回调函数parse
            yield Request(url=page, callback=self.parse)

 这篇博文也写的不错 学习学习https://www.cnblogs.com/jyh-py-blog/p/9986996.html