python-requests爬取数据保存至数据库

import requests
from fake_useragent import UserAgent
from lxml import etree
import pymysql


conn = pymysql.connect(host='47.107.190.1', port=3306, user='zhangsan',
                       password='Changeme_123', database='qiubai', charset='utf8')
# 创建游标
cursor = conn.cursor()

url = 'https://www.qiushibaike.com/text/'

ua = UserAgent()

headers = {
        'User-Agent': ua.random
}

response = requests.get(url=url, headers=headers)
# print(response.text)

# 解析数据
html = etree.HTML(response.text)
# 先拿到存放段子的div
divs = html.xpath('//div[contains(@id, "qiushi_tag_")]')
# print(len(divs))
# 循环从每一个div中提取需要的信息
for div in divs:
    # 定位到具体元素之后, 在当前元素下提取内容,一定要写./
    img = div.xpath('.//div[contains(@class, "author")]/a[1]/img/@src')[0]
    author = div.xpath('string(.//div[contains(@class, "author")]/a[2])').strip()
    detail_url = 'https://www.qiushibaike.com' + div.xpath('.//a[@class="contentHerf"]/@href')[0]
    # print(detail_url)
    # 进入详情页,爬取详情页内容
    detail_response = requests.get(url=detail_url, headers=headers)
    detail_html = etree.HTML(detail_response.text)
    content = detail_html.xpath('string(//div[@class="content"])').strip()
    # print(content)

    # 把爬下来的内容存入到数据库.
    try:
        sql = 'insert into duanzi(img, author, detail_url, content) values("%s", "%s", "%s", "%s")'
        cursor.execute(sql, (img, author, detail_url, content))
        # 提交
        conn.commit()
    except Exception as e:
        print(e)
        conn.rollback()

cursor.close()
conn.close()