前程无忧——数据分析岗位爬取

本文主要是介绍从前程无忧上爬取岗位为数据分析的职位，主要是五个字段，职位名称、公司名称、工作地点、薪资和发布时间。同时把爬取下来的数据保存到mysql数据库中。

import requests
import pymysql
import re


# 连接数据库并创建数据表
db = pymysql.connect('localhost', 'root', 'password', 'lookforjob')
cursor = db.cursor()
cursor.execute('drop table if exists DataAnalyst')
sql = """
create table DataAnalyst
(
PositionName VARCHAR(40),               #职位名称
CompanyName VARCHAR(40),                #公司名称
WorkingPlace VARCHAR(40),               #工作地点
Salary VARCHAR(40),                     #薪资
ReleaseTime VARCHAR(40)                 #发布时间
)
"""

cursor.execute(sql)

def getHTMLText(page):
    url = "https://search.51job.com/list/080200,000000,0000,00,9,99," 
          "%25E6%2595%25B0%25E6%258D%25AE%25E5%2588%2586%25E6%259E%2590%25E5%25B8%2588,2," + str(page) + '.html'
    try:
        r = requests.get(url, timeout=30)
        r.raise_for_status()
        r.encoding = r.apparent_encoding
        html = r.text
        return html
    except:
        return ""


def get(html):
    reg = re.compile(
        r'class="t1 ">.*?<a target="_blank" title="(.*?)".*? <span class="t2"><a target="_blank" title="('r'.*?)".*?<span class="t3">(.*?)</span>.*?<span class="t4">(.*?)</span>.*? <span class="t5">('r'.*?)</span>',
        re.S)
    items = re.findall(reg, html)

    sql = """
        INSERT INTO lookforjob.dataanalyst values(%s,%s,%s,%s,%s)
    """
    for i in range(len(items)):
        cursor.execute(sql, items[i])
        print(items[i])
        db.commit()


if __name__ == "__main__":
    for each in range(1, 7):
        get(getHTMLText(each))
    cursor.close()