假期学习【十一】Python爬取百度词条写入csv格式 python 2020.2.10

今天主要完成了根据爬取的txt文档，从百度分类从信息科学类爬取百度词条信息，并写入CSV格式文件。

txt格式文件如图： 为自己爬取内容分词后的结果。

 代码如下：

import requests
from bs4 import BeautifulSoup
import csv
import io
import re

url="https://baike.baidu.com/item/"
id=1
patton=re.compile(r'.*信息科学分类.*|.*软件.*|.*科技产品.*|.*公司.*|.*互联网人物.*|.*互联网.*|.*科技术语.*|.*技术.*|.*网站.*')

#写入表头
def Head():
    with open('E:/bdbk2.csv', 'w', encoding='utf-8', newline='') as csvfile:
        writer = csv.writer(csvfile)
        writer.writerow(["序号", "名称", "属性", "内容", "网址"])

def read():
    f = open('E:/bdbk2.csv', 'a+', encoding='utf-8', newline='')
    csv_writer = csv.writer(f)
    global url
    f=open("E:/word5.txt",'r+',encoding="utf-8")
    for line in f:
        url=url+line.rstrip("
")
        try:
            global id, name, nature, content, tag
            kv = {'user-agent': 'Mozilla/5.0'}
            r = requests.get(url, headers=kv)
            r.encoding = "utf-8"
            demo = r.text
            soup = BeautifulSoup(demo, "html.parser")
            print(url)
            # print(soup.prettify())
            tag = soup.find_all("dd", {"id": "open-tag-item"})[0].get_text().replace("（", "").replace("）",
                                                                                                      "").strip().replace(
                "
", "")
            name = soup.find_all("h1")[0].get_text().strip()
            nature = soup.find_all("h2")[0].get_text().replace("（", "").replace("）", "").strip()
            if nature == '目录':
                nature = tag
            content = soup.find_all("div", {"class": "lemma-summary"})[0].get_text().strip().rstrip("]").lstrip("[")
            if name != "百度百科错误页" and nature != "目录" and len(patton.findall(tag)) != 0:
                print("序号:" + str(id))
                print("名称:" + name)
                print("属性:" + nature)
                print("内容:" + content)
                print("网址:" + url)
                csv_writer.writerow([str(id), name, nature, content, url])
                id += 1
        except:
            print("出错!")
        url = "https://baike.baidu.com/item/"
    f.close()

if __name__=="__main__":
    Head()
    read()