python3糗事爬取-------------------糗事百科

# -*- coding: utf-8 -*-
#author:zxy
#Date:2018-9-24

import re
import requests

HEADERS = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) '
                  'AppleWebKit/537.36 (KHTML, like Gecko)'
                  ' Chrome/67.0.3396.99 Safari/537.36'
}

def parse_page(url):
    response=requests.get(url,headers=HEADERS)
    text=response.text
    contents=re.findall(r'<divsclass="content">.*?<span>(.*?)</span>',text,re.S)
    for content in  contents:
        x=re.sub(r'<.*?>','
',content) # delete <br>
        with open('qiushi.txt','a',encoding='utf-8') as f:
            f.write(x.strip())
            f.write('
')
            f.write('='*50)
            f.write('
'*3)


def main():
    url="https://www.qiushibaike.com/text/page/1/"
    for x in range(1,4):
        url="https://www.qiushibaike.com/text/page/%s/"%x
        #url="https://www.qiushibaike.com/text/page/2/".format(x)
        parse_page(url)


if __name__ == '__main__':
    main()

效果如下所示：