2021 6 1

顶会热词爬虫：

# -*- coding = UTF-8 -*-
# @Time : 2021/6/12 23:34
# @Author : 伏珊瑞
# @File : test2021.py
# @Software : PyCharm
import urllib
import pymysql
import requests
import jieba.analyse
from bs4 import BeautifulSoup
import re
from pymysql.constants.FIELD_TYPE import JSON
conn = pymysql.connect(host='localhost', user="root", passwd="123456", database="paper")
# 获取游标
cursor = conn.cursor()
Sql="insert into pypaper1(name,herf,writer,Abstract,time,keywords) values(%s,%s,%s,%s,%s,%s)"
def main():  #主函数，之后调用
    html=askURL("https://openaccess.thecvf.com/CVPR2021?day=all")
    getherf(html)
    cursor.close()
    conn.commit()
    conn.close()
def getAbstract(url):   #爬取论文的摘要
    html=askURL(url)       #将网页的html转换成文本
    data=""
    bs = BeautifulSoup(html, "html.parser")
    findlink_herf = re.compile(r'
(.*?)
')#摘要的相应的正则表达式
    a=bs.find_all(id="abstract")
    for item in a:
        item=str(item)
        data=re.findall(findlink_herf,item)[0]
    return data
def getherf(html):#爬取论文所有数据并写入数据库
    bs=BeautifulSoup(html,"html.parser")
    a=bs.find_all(class_="ptitle")
    b = bs.find_all("dd")#dd是网页中每条论文数据的框框
    findlink_herf=re.compile(r'<a href="(.*?)">')#论文的超链接的正则表达式
    findlink_name = re.compile(r'<a href="(.*?)">(.*?)</a></dt>')#论文名字的正则表达式
    findlink_writer = re.compile(r'">(.*?)</a>')#论文作者的正则表达式
    TEMP=1;#爬取的作者连同ptf也爬了所以每两个才是作者数据
    inta=1;#计数器
    for item in a:
        # try:
            item=str(item)
            name=str(b[TEMP])
            link_href=re.findall(findlink_herf,item)[0]
            link_name = re.findall(findlink_name, item)[0]
            writer=re.findall(findlink_writer, name)
            link_writer=""
            for s in writer:
                link_writer+=s+"+"
            link_Abstract=getAbstract("https://openaccess.thecvf.com/"+link_href)#论文超链接应该加上链接头
            keywords=""
            p=0
            for word in jieba.analyse.extract_tags(link_Abstract):#将关键词数组拼成字符串
                keywords+=word+"+"
                p=p+1
                if(p==5):
                    break
            # print(link_name[1])
            # print(link_href)
            # print(link_writer)
            # print(link_Abstract)
            # print(keywords)
            insert = cursor.execute(Sql, (link_name[1], link_href, link_writer, link_Abstract, "2021",keywords))#写入数据库
            TEMP+=2
            print(inta)
            inta+=1
        # except:
        #     print(link_name[1])
        #     print(link_href)
        #     print(link_writer)
        #     print(link_Abstract)
        #     print(keywords)
def askURL(url):#获取网页的html文本
    header = {
        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36'
    }
    request = urllib.request.Request(url, headers=header)
    html = ""
    response = urllib.request.urlopen(request)
    html = response.read().decode("UTF-8")
    return html
main()#调用main函数

# -*- coding = UTF-8 -*-
# @Time : 2021/6/12 23:34
# @Author : 伏珊瑞
# @File : test2021.py
# @Software : PyCharm
import urllib
import pymysql
import requests
import jieba.analyse
from bs4 import BeautifulSoup
import re
from pymysql.constants.FIELD_TYPE import JSON
conn = pymysql.connect(host='localhost', user="root", passwd="123456", database="paper")
# 获取游标
cursor = conn.cursor()
Sql="insert into pypaper(name,herf,writer,Abstract,time,keywords) values(%s,%s,%s,%s,%s,%s)"
def main():
    html=askURL("https://openaccess.thecvf.com/WACV2021")
    getherf(html)
    cursor.close()
    conn.commit()
    conn.close()
    #getAbstract("https://openaccess.thecvf.com/content_WACV_2020/html/Sang_Inferring_Super-Resolution_Depth_from_a_Moving_Light-Source_Enhanced_RGB-D_Sensor_WACV_2020_paper.html")
def getAbstract(url):
    html=askURL(url)
    data=""
    bs = BeautifulSoup(html, "html.parser")
    findlink_herf = re.compile(r'
(.*?)
')
    a=bs.find_all(id="abstract")
    for item in a:
        item=str(item)
        data=re.findall(findlink_herf,item)[0]
    return data
def getherf(html):
    bs=BeautifulSoup(html,"html.parser")
    a=bs.find_all(class_="ptitle")
    b = bs.find_all("dd")
    findlink_herf=re.compile(r'<a href="(.*?)">')
    findlink_name = re.compile(r'<a href="(.*?)">(.*?)</a></dt>')
    findlink_writer = re.compile(r'">(.*?)</a>')
    TEMP=0;
    inta=1;
    for item in a:
        try:
            item=str(item)
            name=str(b[TEMP])
            link_href=re.findall(findlink_herf,item)[0]
            link_name = re.findall(findlink_name, item)[0]
            writer=re.findall(findlink_writer, name)
            link_writer=""
            for s in writer:
                link_writer+=s+"+"
            link_Abstract=getAbstract("https://openaccess.thecvf.com/"+link_href)
            keywords=""
            for word in jieba.analyse.extract_tags(link_Abstract):
                keywords+=word+"+"
            # print(link_name[1])
            # print(link_href)
            # print(link_writer)
            # print(link_Abstract)
            # print(keywords)
            insert = cursor.execute(Sql, (link_name[1], link_href, link_writer, link_Abstract, "2021",keywords))
            #print("insert into pypaper values("+link_name[1]+","+link_href+","+link_writer+","+link_Abstract+",2021)")
            TEMP+=2
            print(inta)
            inta+=1
        except:
            print(link_name[1])
            print(link_href)
            print(link_writer)
            print(link_Abstract)
            print(keywords)
def askURL(url):
    header = {
        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36'
    }
    request = urllib.request.Request(url, headers=header)
    html = ""
    response = urllib.request.urlopen(request)
    html = response.read().decode("UTF-8")
    return html
main()
#def getwriter(html):
#     bs = BeautifulSoup(html, "html.parser")
#     a = bs.find_all("dd")
#     findlink = re.compile(r'">(.*?)</a>')
#     TEMP=2;
#     for item in a:
#         if(TEMP%2==0):
#             item = str(item)
#             link = re.findall(findlink, item)
#             print(link)
#         TEMP+=1
# def getname(html):
#     bs = BeautifulSoup(html, "html.parser")
#     a = bs.find_all(class_="ptitle", )
#     findlink = re.compile(r'<a href="(.*?)">(.*?)</a></dt>')
#     list = bs.find_all(re.compile("dt"))
#     for item in a:
#         item = str(item)
#         link = re.findall(findlink, item)[0]
#         print(link[1])