爬取先知论坛的文章(含图片)

为了一些线下ctf比赛无法联网的情况，特准备了此脚本爬取先知论坛的文章(含图片)。

　　新建xianzhi文件夹，包含如下2个子目录（picture、t）和1个txt文档（exist_tid.txt）：

-rw-r--r--@ 1 shellyzhang staff 0 Sep 11 16:30 exist_tid.txt
drwxr-xr-x 3 shellyzhang staff 96 Sep 11 16:30 picture
drwxr-xr-x 3 shellyzhang staff 96 Sep 11 16:30 t
-rw-r--r-- 1 shellyzhang staff 1993 Sep 15 15:30 xianzhi_sprider.py

　　再在此文件夹下新建脚本xianzhi_sprider.py，内容为：

import requests
import re

#把图片的源地址指向本地
def format_pic_url(old_url):
    old_host = 'https://xzfile.aliyuncs.com/media/upload/picture'
    new_host = '../picture'
    return old_url.replace(old_host,new_host)

#通过文章编号，获取文章内容
def getXianZhiHTML(tid):
    url = 'https://xz.aliyun.com/t/{}'.format(tid)
    resp = requests.get(url)
    # print(resp.status_code)
    if resp.status_code != 200:
        return None
    else:
        return resp.text

#获取图片地址
def getImgUlr(resp_text):
    re_rule = re.compile(r'<img src="https://xzfile.aliyuncs.com.*?">')
    result = []
    for i in re_rule.findall(resp_text):
        result.append(i.replace('<img src="','').replace('">',''))
    return result

#下载图片
def download_img(img_url):
    response = requests.get(img_url)
    img = response.content
    local_img = format_pic_url(img_url).replace('..','.')
    try:
        with open(local_img, 'wb') as f:
            f.write(img)
    except Exception as e:
        print(e)
        pass

#读文件
def readFile(target_file):
    with open(target_file, 'r',encoding='utf-8') as f:
        return f.readlines()

#写文件
def writeFile(target_file,STR):
    with open(target_file, 'a',encoding='utf-8') as f:
        f.write(STR)

#主函数
def main(tid):
    exist_tid = './exist_tid.txt'
    if str(tid)+'
' not in readFile(exist_tid): #之前没有此tid，才进行后续
        resp_text = getXianZhiHTML(tid)
        if resp_text == None:
            pass
        else:
            # 下载所有图片
            for img_url in getImgUlr(resp_text):
                download_img(img_url)
            #保存文章
            t_path = './t/{}.html'.format(tid)
            resp_text = format_pic_url(resp_text)
            writeFile(t_path,resp_text)
            writeFile(exist_tid, str(tid) + '
')
    else:
        print('{} is exist!'.format(tid))

if __name__ == '__main__':
    for tid in range(0,10000):
        main(tid)