爬起点完本小说列表

没来做翻页，只爬第一页

import logging
import requests
import bs4
import json
import psycopg2
from io import StringIO

logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s- %(message)s')

blist = []

def main():
    url = 'https://www.qidian.com/finish/'
    html = request_douban(url)
    soup = bs4.BeautifulSoup(html,'lxml')
    save_to_txt(soup)

def request_douban(url):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) '
                      'Chrome/88.0.4324.146 Safari/537.36',
    }
    try:
        response = requests.get(url,headers=headers)
        if response.status_code == 200:
            return response.text
    except requests.RequestException:
        return None

def save_to_txt(soup):
     booklist = soup.find(class_="main-content-wrap fl").find_all("li")
     logging.info(booklist)
     for i in booklist:
         logging.info(i.find('h4'))
         try:
             shuming = i.find('h4').string
             zuozhe = i.find(class_="name").string
             leixing = i.find(class_="go-sub-type").string
             zhuangtai = i.find(class_="author").find('span').string
             dizhi = 'https:' + i.find('h4').find( 'a', href=True)['href']

             tap = (shuming, zuozhe, leixing, zhuangtai, dizhi)
             blist.append('	'.join('%s' % idd for idd in tap))
             logging.info(list)
             writedatabase(list)
         except:
             return None


def writedatabase(blist):
    with open("config/kafka_and_postgres_config.json") as json_file:
        config = json.load(json_file)
    Conf = json.loads(json.dumps(config))
    pghost = Conf['postgres_host']
    pguser = Conf['postgres_user']
    pgpassword = Conf['postgres_password']
    pgdatabase = Conf['postgres_database']
    pgtable = Conf['postgres_table']

    s = ""
    for action in blist:
        s += action + '
'
        logging.info(s)

    conn = psycopg2.connect(host=pghost, user=pguser, password=pgpassword, database=pgdatabase)
    cur = conn.cursor()
    cur.copy_from(StringIO(s), pgtable, columns=('shuming', 'zuozhe', 'leixing', 'zhuangtai', 'dizhi'))
    conn.commit()
    cur.close()
    conn.close()
    list = []
    print('完成')


if __name__=="__main__":
    main()