scrapy爬虫框架

scrapy爬虫框架爬取百度贴吧帖子并存库

参考：http://cuiqingcai.com/3472.html/3

scrapySpiders.mysqlpipelines.pipelines.ScrapyPipelines

# -*- coding: utf-8 -*-
__author__ = 'mmc'

from sql import Sql
from scrapySpiders.items import ScrapyspidersItem


class ScrapyPipelines(object):
    @classmethod
    def process_item(self, item, spider):
        if isinstance(item, ScrapyspidersItem):
            print "start save data in database"
            tb_name = item['name']
            tb_content = item['content']
            Sql.insert_content(tb_name, tb_content)

在settings.py文件中修改配置即可自动进行数据存储

spiders

# -*- coding: utf-8 -*-
__author__ = 'mmc'

import re
import scrapy
from bs4 import BeautifulSoup
from scrapy.http import Request
from scrapySpiders.items import ScrapyspidersItem
from scrapySpiders.mysqlpipelines.pipelines import ScrapyPipelines


class Myspiders(scrapy.Spider):

    name = 'TieBa'
    base_url = 'https://tieba.baidu.com/p/'

    def start_requests(self):
        url = self.base_url+'4420237089'+'?see_lz=1'
        yield Request(url, callback=self.parse)

    def parse(self, response):
        max_num = BeautifulSoup(response.text, 'lxml').find('li', class_='l_reply_num').find_all('span')[-1].get_text()
        print max_num
        # print response.text
        baseurl = str(response.url)
        for num in range(1, int(max_num)+1): #int(max_num)+1
            url = baseurl+'&pn='+str(num)
            # print url
            yield Request(url, callback=self.get_name)

    def get_name(self, response):
        response = BeautifulSoup(response.text, 'lxml')
        item = ScrapyspidersItem()
        tmp = response.findAll("div", {"class": "d_post_content j_d_post_content "})
        vmp = response.findAll("span", {"class": "tail-info"})
        for val, f in zip(tmp, vmp[1:-1:3]):
            # print val.get_text()
            # print f.get_text()
            item['name'] = val.get_text().strip()
            item['content'] = f.get_text().strip()
            print item
            ScrapyPipelines().process_item(item,'')
        return item
        # item['name'] = tmp
        # item['content'] = vmp
        # # print tmp
        # # print vmp
        # return item

sql.py

# -*- coding: utf-8 -*-
__author__ = 'mmc'

import MySQLdb


db_para = {'host': '127.0.0.1',
           'port': 3306,
           'user': 'mmc',
           'passwd': 'mmc123456',
           'db': 'test'}
dbcon = MySQLdb.connect(**db_para)
cur = dbcon.cursor()
#出现UnicodeEncodeError: 'latin-1' codec can't encode character执行以下代码解决
dbcon.set_character_set('utf8')
cur.execute('SET NAMES utf8;')
cur.execute('SET CHARACTER SET utf8;')
cur.execute('SET character_set_connection=utf8;')


class Sql:

    @classmethod
    def insert_content(cls, tb_name, tb_content):

        sql = """INSERT INTO `test`.`tb_name` (`tb_name`, `tb_content`) VALUES (%(tb_content)s,%(tb_name)s) """
        value = {'tb_name': tb_name,
                 'tb_content': tb_content}
        cur.execute(sql, value)
        dbcon.commit()

scrapy查询语法：

当我们爬取大量的网页，如果自己写正则匹配，会很麻烦，也很浪费时间，令人欣慰的是，scrapy内部支持更简单的查询语法，帮助我们去html中查询我们需要的标签和标签内容以及标签属性。下面逐一进行介绍：

• 查询子子孙孙中的某个标签(以div标签为例)：//div
• 查询儿子中的某个标签(以div标签为例)：/div
• 查询标签中带有某个class属性的标签：//div[@class=’c1′]即子子孙孙中标签是div且class=‘c1’的标签
• 查询标签中带有某个class=‘c1’并且自定义属性name=‘alex’的标签：//div[@class=’c1′][@name=’alex’]
• 查询某个标签的文本内容：//div/span/text() 即查询子子孙孙中div下面的span标签中的文本内容
• 查询某个属性的值（例如查询a标签的href属性）：//a/@href

示例代码：

from scrapy.selector import Selector
from scrapy.http import HtmlResponse

def parse(self, response):
       # 分析页面
       # 找到页面中符合规则的内容（校花图片），保存
       # 找到所有的a标签，再访问其他a标签，一层一层的搞下去
 
       hxs = HtmlXPathSelector(response)#创建查询对象
 
       # 如果url是 http://www.xiaohuar.com/list-1-d+.html
       if re.match('http://www.xiaohuar.com/list-1-d+.html', response.url): #如果url能够匹配到需要爬取的url，即本站url
           items = hxs.select('//div[@class="item_list infinite_scroll"]/div') #select中填写查询目标，按scrapy查询语法书写
           for i in range(len(items)):
               src = hxs.select('//div[@class="item_list infinite_scroll"]/div[%d]//div[@class="img"]/a/img/@src' % i).extract()#查询所有img标签的src属性，即获取校花图片地址
               name = hxs.select('//div[@class="item_list infinite_scroll"]/div[%d]//div[@class="img"]/span/text()' % i).extract() #获取span的文本内容，即校花姓名
               school = hxs.select('//div[@class="item_list infinite_scroll"]/div[%d]//div[@class="img"]/div[@class="btns"]/a/text()' % i).extract() #校花学校
               if src:
                   ab_src = "http://www.xiaohuar.com" + src[0]#相对路径拼接
                   file_name = "%s_%s.jpg" % (school[0].encode('utf-8'), name[0].encode('utf-8')) #文件名，因为python27默认编码格式是unicode编码，因此我们需要编码成utf-8
                   file_path = os.path.join("/Users/wupeiqi/PycharmProjects/beauty/pic", file_name)
                   urllib.urlretrieve(ab_src, file_path)