爬取前尘无忧python职位信息并保存到mongo数据库

１．re实现

import re,os
import requests
from requests.exceptions import RequestException

MAX_PAGE = 10 #最大页数
KEYWORD = 'python'
headers = {
    'User-Agent':
        'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'
}
file_name = 're_job51_python.txt'

# 获取网页源码
def getHtml(page):
    try:
        url = 'https://search.51job.com/list/040000,000000,0000,00,9,99,{0},2,{1}.html?'.format(KEYWORD,page)
        response = requests.get(url,headers=headers)
        response.encoding = response.apparent_encoding
        return response.text
    except RequestException:
        print('请求出错')
        return None

# 解析网页源码，得到目标信息
def getTarget(html):
    reg = re.compile(
        r'class="t1 ">.*? <a target="_blank" '
        'title="(.*?)".*? <span class="t2"><a target="_blank" '
        'title="(.*?)".*?<span '
        'class="t3">(.*?)</span>.*?<span '
        'class="t4">(.*?)</span>.*? <span '
        'class="t5">(.*?)</span>',
        re.S)  # 匹配换行符
    target = re.findall(reg,html)
    return target


# 保存到文本中
def save_to_txt(item):
    with open(file_name,'a',newline='') as f:  # newline参数防止两行之间有空行
        for i in range(len(item)):
            # 最后一个元素换行，非最后则以＇,'隔开
            if i == len(item)-1:
                f.write(item[i])
                f.write('
')
            else:
                f.write(item[i]+',')

def main():
    # 每次执行前检查文件是否存在，存在则删除
    if os.path.exists(file_name):
        os.remove(file_name)

    # 分页爬取
    for page in range(MAX_PAGE+1):
        html = getHtml(page)
        content = getTarget(html)
        for item in content:
            save_to_txt(item)

if __name__ == '__main__':
    main()

 ２．xpath实现

import os
import requests
from requests.exceptions import RequestException
from lxml import etree
import pymongo
from spiders.前程无忧.mongo_config import *

# mongo数据库设置
client = pymongo.MongoClient(MONGO_URL)
db = client[MONGO_DB]

MAX_PAGE = 5
KEYWORD = 'python'
headers = {
    'User-Agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) '
    'Chrome/63.0.3239.132 Safari/537.36'
}
file_name = 'xpath_job51_python.txt'

# 获取网页
def get_html(page):
    try:
        url = 'https://search.51job.com/list/040000,000000,0000,00,9,99,{},2,{}.html?'.format(KEYWORD,page)
        response = requests.get(url,headers=headers)
        response.encoding = response.apparent_encoding
        return response.text
    except RequestException:
        return None

# 解析网页
def parse_html(html):
    # 构造xpath解析对象，可自动修整HTML文本
    html = etree.HTML(html)
    # 获取文本 /text()
    # 获取属性　／@href
    # 获取第i个标签　／tar_name[i]  从1开始
    # normalize-space－－＞去空格换行符
    # position_name = html.xpath('normalize-space(//div[@class="el"]/p/span/a/text())')

    # 职位名称，
    position_names = []
    for name in html.xpath('//div[@class="el"]/p/span/a/text()'):
        position_name = name.strip()
        position_names.append(position_name)

    # 职位地址
    position_urls = html.xpath('//div[@class="el"]/p/span/a/@href')

    # 公司名称
    company_names = html.xpath('//div[@class="el"]/span[1]/a/text()')

    # 公司地址
    company_urls = html.xpath('//div[@class="el"]/span[1]/a/@href')

    # 位置
    locations = html.xpath('//div[@class="el"]/span[@class="t3"]/text()')

    # 薪资
    salarys = html.xpath('//div[@class="el"]/span[@class="t4"]/text()')

    # 发布时间
    release_dates = html.xpath('//div[@class="el"]/span[4]/text()')

    result = zip(position_names,position_urls,company_names,company_urls,locations,salarys,release_dates)
    return result


def save_to_txt(element):
    with open(file_name,'a',newline='') as f:
        for i in range(len(element)):
            # data = ','.join(element[i])
            if i == len(element)-1:
                f.write(element[i])
                f.write('
')
            else:
                f.write(element[i]+',')


def save_to_mongo(element):
    keys = ['position_name','position_url','company_name',
            'company_url','location','salary','release_date']
    result = dict(zip(keys,list(element)))
    if db[MONGO_TABLE_XPATH].insert(result):
        print('数据成功存储到mongo数据库中')
        return True
    return False

    # 遍历字典元素
    # for k,v in result.items():
    #     print(k,':',v)
    for key in result:
        print(key,':',result[key])



def main():
    if os.path.exists(file_name):
        os.remove(file_name)
    for page in range(1,MAX_PAGE+1):
        html = get_html(page)
        elements = parse_html(html)
        if elements:
            for element in elements:
                save_to_txt(element)
                save_to_mongo(element)

if __name__ == '__main__':
    main()