scrapy 反扒措施_2

目录

• 措施实现：
  • 禁用Cookie：
  • .设置下载延时：
  • 设置USER-AGENT和代理ip:
  • 对接 Selenium:

措施实现：

禁用Cookie：

Scrapy中，默认是打开cookie的 （#COOKIES_ENABLED = False）
	设置为：COOKIES_ENABLED = False （cookie启用：no），对于需要cookie的可以在请求头中headers加入cookie
    
class LagouspiderSpider(scrapy.Spider):
    name = "lagouspider"
    allowed_domains = ["www.lagou.com"]
 
    url = 'https://www.lagou.com/jobs/positionAjax.json?'#city=%E6%B7%B1%E5%9C%B3&needAddtionalResult=false'
    page = 1
    allpage =0
 
    cookie = 'JSESSIONID=ABAAABAAAFCAAEG34858C57541C1F9DF75AED18C3065736; Hm_lvt_4233e74dff0ae5bd0a3d81c6ccf756e6=1524281748;  04797acf-4515-11e8-90b5- LGSID=20180421130026-e7e614d7-4520-PRE_SITE=https%3A%2F%2Fwww.lagou.com%2Fjobs%2Flist_python%3Fcity%3D%25E6%25B7%25B1%25E5%259C%25B3%26cl%3Dfalse%26fromSearch%3Dtrue%26labelWords%3D%26suginput%3D; PRE_LAND=https%3A%2F%2Fwww.lagou.com%2Fjobs%2F4302345.html; LGRID=20180421130208-24b73966-4521-11e8-90f2-525400f775ce; Hm_lpvt_4233e74dff0ae5bd0a3d81c6ccf756e6=1524286956'
    headers = {'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
               'Referer': 'https://www.lagou.com/jobs/list_python?city=%E6%B7%B1%E5%9C%B3&cl=false&fromSearch=true&labelWords=&suginput=',
               'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36',
               'cookie': cookie }
 
    def start_requests(self):
       yield scrapy.FormRequest(self.url, headers=self.headers, formdata={
            'first': 'true','pn': str(self.page),'kd': 'python','city': '深圳'}, callback=self.parse)

    --> post 请求

.设置下载延时：

在Scrapy中，默认是关闭请求下载延时的（#DOWNLOAD_DELAY = 3）

去掉#，或者在spider的请求间中加入 time.sleep(random.randint(5, 10))

def parse(self, response):
        #print(response.text)
        item = LagouItem()
        data = json.loads(response.text)
 
        totalCount = data['content']['positionResult']['totalCount']#总共多少条信息
        resultSize = data['content']['positionResult']['resultSize']#每页多少条信息
 
        result = data['content']['positionResult']['result']#得到一个包含15个信息的列表
        for each in result:
            for field in item.fields:
                if field in each.keys():
                    item[field] = each.get(field)
            yield item
 
        time.sleep(random.randint(5, 10))
 
        if int(resultSize):
            self.allpage = int(totalCount) // int(resultSize) + 1
            if self.page < self.allpage:
                self.page += 1
                yield scrapy.FormRequest(self.url, headers=self.headers, formdata={
            'first': 'false','pn': str(self.page),'kd': 'python','city': '深圳'}, callback=self.parse)

设置USER-AGENT和代理ip:

settings.py:
DOWNLOADER_MIDDLEWARES = {
    'doubanMongo.middlewares.RandomUserAgent': 300,
    'doubanMongo.middlewares.RandomProxy':400
}
 
USER_AGENTS = [
    'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0)',
    'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.2)',
    'Opera/9.27 (Windows NT 5.2; U; zh-cn)',
    'Opera/8.0 (Macintosh; PPC Mac OS X; U; en)',
    'Mozilla/5.0 (Macintosh; PPC Mac OS X; U; en) Opera 8.0',
    'Mozilla/5.0 (Linux; U; Android 4.0.3; zh-cn; M032 Build/IML74K) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30',
    'Mozilla/5.0 (Windows; U; Windows NT 5.2) AppleWebKit/525.13 (KHTML, like Gecko) Chrome/0.2.149.27 Safari/525.13'
]
 
PROXIES=[{'ip_port':'117.48.214.249:16817','user_passwd':'632345244:4tf9pcpw'}
        #{'ip_port':'117.48.214.249:16817','user_passwd':''},
        #{'ip_port':'117.48.214.249:16817','user_passwd':''},
        #{'ip_port':'117.48.214.249:16817','user_passwd':''}
       ]

middlewares中：

from scrapy.conf import settings
import base64
import random
 
 
class RandomProxy(object):
 
    def process_request(self, request, spider):
        proxy = random.choice(settings["PROXIES"])
        if proxy['user_passwd'] is None:
            request.meta['proxy'] = 'http://'+ proxy['ip_port']
        else:
            # 对账户密码进行base64编码转换
            b_pw =  bytes(proxy['user_passwd'], encoding = "utf-8")#string转为bytes
            base64_userpasswd = base64.encodestring(b_pw)#需要的参数是bytes对象
            # 对应到代理服务器的信令格式里
            s_base64_userpasswd = str(base64_userpasswd, encoding="utf-8") #bytes转为string
            request.headers['Proxy-Authorization'] = 'Basic ' + s_base64_userpasswd
            request.meta['proxy'] = "http://" + proxy['ip_port']
 
 
class RandomUserAgent(object):
 
    def process_request(self, request, spider):
        useragent = random.choice(settings["USER_AGENTS"])
        #print(useragent)
        request.headers.setdefault('User-Agent',useragent)

Selenium scrapy:

PhantomJS:
    	https://phantomjs.org/download.html
点击进入phantomjs-2.1.1-windows文件夹，进入bin文件夹，复制路径

添加到环境变量： 启动（phantomjs -v,）

执行： phantomjs.exe

pip install Selenium,PyMongo

scrapy startproject scrapyseleniumtest
scrapy genspider taobao www.taobao.com

from scrapy import Item, Field
class ProductItem(Item):
   collection = 'products'
   image = Field()
   price = Field()
   deal = Field()
   title = Field()
   shop = Field()
   location = Field()

from scrapy import Request, Spider
from urllib.parse import quote
from scrapyseleniumtest.items import ProductItem
 
class TaobaoSpider(Spider):
   name = 'taobao'
   allowed_domains = ['www.taobao.com']
   base_url = 'https://s.taobao.com/search?q='
 
   def start_requests(self):
       for keyword in self.settings.get('KEYWORDS'):
           for page in range(1, self.settings.get('MAX_PAGE') + 1):
               url = self.base_url + quote(keyword)
               yield Request(url=url, callback=self.parse, meta={'page': page}, dont_filter=True)

setttings.py:

KEYWORDS = ['iPad']
MAX_PAGE = 100

DOWNLOADER_MIDDLEWARES = {
   'scrapyseleniumtest.middlewares.SeleniumMiddleware': 543,
}

ITEM_PIPELINES = {
   'scrapyseleniumtest.pipelines.MongoPipeline': 300,
}
MONGO_URI = 'localhost'
MONGO_DB = 'taobao'

对接 Selenium:

from selenium import webdriver
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from scrapy.http import HtmlResponse
from logging import getLogger
 
class SeleniumMiddleware():
   def __init__(self, timeout=None, service_args=[]):
       self.logger = getLogger(__name__)
       self.timeout = timeout
       self.browser = webdriver.PhantomJS(service_args=service_args)
       self.browser.set_window_size(1400, 700)
       self.browser.set_page_load_timeout(self.timeout)
       self.wait = WebDriverWait(self.browser, self.timeout)
 
   def __del__(self):
       self.browser.close()
 
   def process_request(self, request, spider):
       """
       用PhantomJS抓取页面
       :param request: Request对象
       :param spider: Spider对象
       :return: HtmlResponse
       """
       self.logger.debug('PhantomJS is Starting')
       page = request.meta.get('page', 1)
       try:
           self.browser.get(request.url)
           if page > 1:
               input = self.wait.until(
                   EC.presence_of_element_located((By.CSS_SELECTOR, '#mainsrp-pager div.form > input')))
               submit = self.wait.until(
                   EC.element_to_be_clickable((By.CSS_SELECTOR, '#mainsrp-pager div.form > span.btn.J_Submit')))
               input.clear()
               input.send_keys(page)
               submit.click()
           self.wait.until(EC.text_to_be_present_in_element((By.CSS_SELECTOR, '#mainsrp-pager li.item.active > span'), str(page)))
           self.wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, '.m-itemlist .items .item')))
           return HtmlResponse(url=request.url, body=self.browser.page_source, request=request, encoding='utf-8', status=200)
       except TimeoutException:
           return HtmlResponse(url=request.url, status=500, request=request)
 
   @classmethod
   def from_crawler(cls, crawler):
       return cls(timeout=crawler.settings.get('SELENIUM_TIMEOUT'),
                  service_args=crawler.settings.get('PHANTOMJS_SERVICE_ARGS'))
    
    -->利用PhantomJS来代替Scrapy完成了页面的加载，最后将Response返回

解析页面：

def parse(self, response):
   products = response.xpath(
       '//div[@id="mainsrp-itemlist"]//div[@class="items"][1]//div[contains(@class, "item")]')
   for product in products:
       item = ProductItem()
       item['price'] = ''.join(product.xpath('.//div[contains(@class, "price")]//text()').extract()).strip()
       item['title'] = ''.join(product.xpath('.//div[contains(@class, "title")]//text()').extract()).strip()
       item['shop'] = ''.join(product.xpath('.//div[contains(@class, "shop")]//text()').extract()).strip()
       item['image'] = ''.join(product.xpath('.//div[@class="pic"]//img[contains(@class, "img")]/@data-src').extract()).strip()
       item['deal'] = product.xpath('.//div[contains(@class, "deal-cnt")]//text()').extract_first()
       item['location'] = product.xpath('.//div[contains(@class, "location")]//text()').extract_first()
       yield item

存储：

import pymongo
 
class MongoPipeline(object):
   def __init__(self, mongo_uri, mongo_db):
       self.mongo_uri = mongo_uri
       self.mongo_db = mongo_db
 
   @classmethod
   def from_crawler(cls, crawler):
       return cls(mongo_uri=crawler.settings.get('MONGO_URI'), mongo_db=crawler.settings.get('MONGO_DB'))
 
   def open_spider(self, spider):
       self.client = pymongo.MongoClient(self.mongo_uri)
       self.db = self.client[self.mongo_db]
 
   def process_item(self, item, spider):
       self.db[item.collection].insert(dict(item))
       return item
 
   def close_spider(self, spider):
       self.client.close()

main.py

from scrapy import cmdline

cmdline.execute('scrapy crawl taobao --nolog '.split())