python+urllib+BeautifulSoup+pymysql

# -*- coding: utf-8 -*-
# @Time : 2019/11/12 21:22
# @Author : AForever
# @Site : 
# @File : cnblog_002.py
# @Software: PyCharm

from urllib import request
from bs4 import BeautifulSoup
import os
import pymysql


# 获取数据
def get_data():
    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36"
    }

    for i in range(7):
        url = "https://search.51job.com/list/040000,000000,0000,00,9,99,python%25E5%25BC%2580%25E5%258F%2591,2," + str(i+1) + ".html"
        req = request.Request(url, headers=headers)
        file_name = 'E:\python_project\Spider\cnblogs\data\cnblog_pythonjob' + str(i + 1) + '.html'
        print(file_name, url)
        response = request.urlopen(req)

        if response.getcode() == 200:
            data = response.read()
            data = str(data, encoding='gbk')
            with open(file_name, mode="w", encoding="gbk") as f:
                f.write(data)
    print("*"*15, "get data success", "*"*15)


# 解析数据
def parse_data():
    path = 'E:\python_project\Spider\cnblogs\data\'
    filenames = os.listdir(path)
    result = []
    for filename in filenames:
        # print(filename)
        with open(path+filename, mode="r", encoding="gbk") as f:
            html = f.read()
            bs = BeautifulSoup(html, 'html.parser')
            divs = bs.select('#resultList .el')
            for div in divs[1:]:
                title = div.select('.t1')[0].get_text(strip=True)
                company = div.select('.t2')[0].get_text(strip=True)
                addr = div.select('.t3')[0].get_text(strip=True)
                salary = div.select('.t4')[0].get_text(strip=True)
                pubdate = div.select('.t5')[0].get_text(strip=True)
                row = {
                    'title': title,
                    'company': company,
                    'addr': addr,
                    'salary': salary,
                    'pubdate': pubdate
                }
                result.append(row)
        # print(result)

    print('*' * 15, 'parse data success, ,Congratulations!', '*' * 15)
    # print(result)
    return result


# 创建数据表
def create_table():
    config = {
        'host': 'localhost',
        'port': 3306,
        'user': 'root',
        'password': '123456',
        'database': 'python',
        'charset': 'utf8'
    }
    conn = pymysql.connect(**config)
    cursor = conn.cursor()
    # 如果存在student表，则先删除
    try:
        cursor.execute('DROP TABLE IF EXISTS `t_job`;')
        conn.commit()
        print('*' * 15, "drop table success", '*' * 15)
    except:
        print('*' * 15, 'table dose not exist', '*' * 15)

    create_table = '''
        create table t_job(
        id int primary key auto_increment,
        title varchar(200),
        company varchar(200),
        addr varchar(200),
        salary varchar(200),
        pubdate varchar(200)
        )engine=Innodb charset utf8;
        '''
    # 创建数据表
    cursor.execute(create_table)
    cursor.close()
    conn.close()
    print('*' * 15, 'create tables success,Congratulations!', '*' * 15)


# 存储数据到mysql
def save_to_mysql(data):
    config = {
        'host': 'localhost',
        'port': 3306,
        'user': 'root',
        'password': 'lem600@HW',
        'database': 'python',
        'charset': 'utf8'
    }

    conn = pymysql.connect(**config)
    cursor = conn.cursor()
    sql = '''
    insert into t_job(title, company, addr, salary, pubdate)
    values(%(title)s,%(company)s,%(addr)s,%(salary)s,%(pubdate)s)
    '''
    cursor.executemany(sql, data)
    conn.commit()
    cursor.close()
    conn.close()
    print('*' * 15, 'save data to mysql success ,Congratulations !', '*' * 15)


if __name__ == "__main__":
    get_data()
    # parse_data()
    create_table()
    save_to_mysql(parse_data())