中国大学排名定向爬虫

本篇爬虫主要是为了从最好大学网站上爬取2019年各个大学的排名，以及把数据存储到mysql的过程：

import requests
from bs4 import BeautifulSoup
import bs4
import pymysql

# 连接数据库并且创建数据表
db = pymysql.connect('localhost', 'root', 'password', 'universityrankings')
cursor = db.cursor()
cursor.execute('drop table if exists UNRANKING2019')
sql = """
create table UNRANKING2019
(
paiming INTEGER,
xuexiaomingchen VARCHAR(40),
shengshi VARCHAR(40),
zongfen VARCHAR(40),
shengyuanzhiliang VARCHAR(40),
peiyangjieguo VARCHAR(40),
shehuishengyu VARCHAR(40),
keyanguimo VARCHAR(40),
keyanzhiliang VARCHAR(40),
dingjianchengguo VARCHAR(40),
dingjianrencai VARCHAR(40),
kejifuwu VARCHAR(40),
chengguozhuanhua VARCHAR(40),
xueshengguojihua VARCHAR(40),
primary key(xuexiaomingchen)
);
"""
cursor.execute(sql)


def getHTMLText(url):
    try:
        r = requests.get(url, timeout=30)
        r.raise_for_status()
        r.encoding = r.apparent_encoding
        return r.text
    except:
        return ""


def fillUnivlist(ulist, html):
    soup = BeautifulSoup(html, "html.parser")
    for tr in soup.find('tbody').children:
        if isinstance(tr, bs4.element.Tag):
            tds = tr.find_all('td')
            ulist.append([tds[0].string, tds[1].string, tds[2].string, tds[3].string, tds[4].string, tds[5].string,
                          tds[6].string, tds[7].string, tds[8].string, tds[9].string, tds[10].string, tds[11].string,
                          tds[12].string, tds[13].string])
    sql = """
        INSERT INTO universityrankings.unranking2019 values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)
        """
    for i in range(len(ulist)):
        cursor.execute(sql, ulist[i])
    db.commit()
    cursor.close()


def printUnivList(ulist, num):
    tplt = "{0}	{1}	{2}	{3}	{4}	{5}	{6}	{7}	{8}	{9}	{10}	{11}	{12}	{13}"
    print(tplt.format("排名", "学校名称", "省市", "总分", "生源质量", "培养结果", "社会声誉", "科研规模", "科研质量", "顶尖成果", "顶尖人才", "科技服务", "成果转化",
                      "学生国际化"))
    for i in range(num):
        u = ulist[i]
        print(tplt.format(u[0], u[1], u[2], u[3], u[4], u[5], u[6], u[7], u[8], u[9], u[10], u[11], u[12], u[13]))


def main():
    uinfo = []
    url = 'http://www.zuihaodaxue.com/zuihaodaxuepaiming2019.html'
    html = getHTMLText(url)
    fillUnivlist(uinfo, html)
    printUnivList(uinfo, 549)


main()