PubMed数据下载

目标站点分析

目标：抓取页面中的机构名称，日期，标题，作者， 作者信息， 摘要

程序实现

# -*- coding: utf-8 -*-

"""
@Datetime: 2019/3/6
@Author: Zhang Yafei
"""
import os
import re
import time
from concurrent.futures import ThreadPoolExecutor
import traceback

import pandas as pd
import requests
from pyquery import PyQuery as pq

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36'}

BASE_DIR = 'html'

if not os.path.exists(BASE_DIR):
    os.mkdir(BASE_DIR)


class PubMed(object):
    def __init__(self, url):
        self.url = url
        # self.url = 'https://www.ncbi.nlm.nih.gov/pubmed/{}'.format(id)
        self.retry = 0

    def download(self):
        try:
            response = requests.get(self.url, headers=headers, timeout=20)
            if response.status_code == 200:
                self.parse(response.content)
        except Exception as e:
            traceback.print_exc()
            print('error:' + self.url)
            while True:
                self.retry += 1
                if self.retry < 5:
                    try:
                        response = requests.get(self.url, headers=headers, timeout=15)
                        if response.status_code == 200:
                            self.parse(response.content)
                            return
                    except Exception as e:
                        print(e)
                        time.sleep(10)
                else:
                    print(self.url + '下载失败')
                    return

    def parse(self, response):
        doc = pq(response, parser='html')
        periodical_item = doc('.cit')
        periodical = periodical_item.children().text()
        try:
            periodical_datetime = re.search('</a>(.*?);', periodical_item.__str__()).group(1)
        except AttributeError:
            periodical_datetime = re.search('</a>(.*?).', periodical_item.__str__()).group(1)
        title = doc('.rprt_all h1').text()
        authors_items = doc('.auths a').items()
        authors = ','.join(list(map(lambda x: x.text(), authors_items)))
        author_info = doc('.ui-ncbi-toggler-slave dd').text()
        abstract = doc('.abstr').text()
        data_dict = {'url': [self.url], 'periodical': [periodical], 'periodical_datetime': [periodical_datetime],
                     'title': [title], 'authors': [authors], 'author_info': [author_info], 'abstract': [abstract]}
        self.write_csv(filename='pubmed_result.csv', data=data_dict)
        print(self.url + '下载完成')

    @staticmethod
    def write_csv(filename, data=None, columns=None, header=False):
        """ 写入 """
        if header:
            df = pd.DataFrame(columns=columns)
            df.to_csv(filename, index=False, mode='w')
        else:
            df = pd.DataFrame(data=data)
            df.to_csv(filename, index=False, header=False, mode='a+')


def filter_url_list(urls_list):
    df = pd.read_csv('pubmed_result.csv')
    has_urls = df.url.tolist()
    url_list = set(urls_list) - set(has_urls)
    print('共：{} 完成：{} 还剩：{}'.format(len(urls_list), len(has_urls), len(url_list)))
    return list(url_list)


def read_data():
    df = pd.read_excel('data.xlsx', header=None)
    return df[0].tolist()


def main(url):
    """ 主函数 """
    pubmed = PubMed(url=url)
    pubmed.download()


if __name__ == '__main__':
    url_list = read_data()
    if not os.path.exists('pubmed_result.csv'):
        columns = ['url', 'periodical', 'periodical_datetime', 'title', 'authors', 'author_info', 'abstract']
        PubMed.write_csv(filename='pubmed_result.csv', columns=columns, header=True)
    else:
        url_list = filter_url_list(url_list)

    pool = ThreadPoolExecutor()
    pool.map(main, url_list)
    pool.shutdown()

    # 写入excel
    df = pd.read_csv('pubmed_result.csv')
    writer = pd.ExcelWriter('pubmed_result.xlsx')
    df.to_excel(writer, 'table', index=False)
    writer.save()