zoukankan      html  css  js  c++  java
  • 爬取新闻列表

    import requests
    import re
    from bs4 import BeautifulSoup
    from datetime import datetime
    
    gzccurl='http://news.gzcc.cn/html/xiaoyuanxinwen/'
    res=requests.get(gzccurl)
    res.encoding='utf-8'
    soup = BeautifulSoup(res.text,'html.parser')
    li =soup.select('li')
    
    def getclick(newsurl):
        id=re.search('_(.*).html',newsurl).group(1).split('/')[1]
        clickurl='http://oa.gzcc.cn/api.php?op=count&id={}&modelid=80'.format(id)
        click=int(requests.get(clickurl).text.split('.')[-1].lstrip("html('").rstrip("');"))
        return click
    
    def getonepage(listurl):
        for news in soup.select('li'):
            if len(news.select('.news-list-title'))>0:
                title=news.select('.news-list-title')[0].text
                url=news.select('a')[0]['href']
                time=news.select('.news-list-info')[0].contents[0].text
                dt=datetime.strptime(time,'%Y-%m-%d')
                source=news.select('.news-list-info')[0].contents[1].text
    
                resd=requests.get(url)
                resd.encoding='utf-8'
                soupd=BeautifulSoup(resd.text,'html.parser')
                detail=soupd.select('.show-content')[0].text
                click=getclick(url)
                print(time,title,dt,source,click)
                
    getonepage(li)
    
    pages=int(soup.select('.a1')[0].text.rstrip(''))//10+1
    
    for i in range(2,pages+1):
        url_page="http://news.gzcc.cn/html/xiaoyuanxinwen/{}.html".format(i)
        res_page=requests.get(url_page)
        res_page.encoding='utf-8'
        soup_page=BeautifulSoup(res.text,'html.parser')
        list_page=soup.select('li')
        getonepage(list_page)

  • 相关阅读:
    数据库使用动态监听导致EM起不来的解决方法
    OCP-1Z0-053-V12.02-115题
    OCP-1Z0-053-V12.02-150题
    OCP-1Z0-053-V12.02-136题
    OCP-1Z0-053-V12.02-154题
    OCP-1Z0-053-V12.02-149题
    OCP-1Z0-053-V12.02-146题
    OCP-1Z0-053-V12.02-160题
    OCP-1Z0-053-V12.02-157题
    OCP-1Z0-053-V12.02-164题
  • 原文地址:https://www.cnblogs.com/lqy-36/p/7652548.html
Copyright © 2011-2022 走看看