1、在巨人的肩膀上，结合网上的资源，梳理出来的。
2、其中应用了几个常用的包，requests、re等，
3、注意创建多级文件夹要用--makesdir，创建单级文件用--mkdir
1 # 导入相应的包 2 # 请求网页 3 import requests 4 # 正则解析网页 5 import re 6 # 告诉服务，自己的身份， 7 import time 8 9 import os 10 11 # 函数请求的头部信息 12 headers = { 13 "user-agent": 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.114 Safari/537.36 Edg/89.0.774.68' 14 } 15 16 ############################################ 开始编写程序 ############################################ 17 # 定义轮询的页面数 18 polling_page_num = int(input("请输入主页页码轮询的数量:")) 19 20 ############################################ 函数区 ############################################ 21 22 # 下载界面html代码的函数 23 def download_page(url, html_encode='utf-8', *args, **kwargs): 24 """ 25 下载界面html代码的函数 26 :param url:需要下载网页代码的链接 27 :param html_encode:网页的解码方式。默认是“utf-8” 28 :param args: 29 :param kwargs: 30 :return:返回值为该页面的html代码 31 """ 32 headers = { 33 "user-agent": 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.114 Safari/537.36 Edg/89.0.774.68'} 34 r = requests.get(url, headers=headers) 35 r.encoding = html_encode 36 response_status = r.status_code 37 return r.text, response_status 38 39 40 # 分析主网页并且返回已经处理后的链接列表 41 def anasy_main_page(html, *args, **kwargs): 42 ex = '<a href="(.*?)" title=".*?"><img alt=".*?" src=".*?"><i>.*?</i></a>' 43 # ex = '<a href="(.*?)" title=.*?><img alt=.*? src=.*?><i>.*?</i></a>' 44 test_src_list = re.findall(ex, html, re.S) 45 new_src_list = test_src_list[1:31] 46 li_piclink = [] 47 for pic_link in new_src_list: 48 new_pic_link = '链接' + pic_link 49 li_piclink.append(new_pic_link) 50 return li_piclink 51 52 53 # 分析副网页,返回图片的下载地址 54 def anasy_Secondary_page(Secondary_html): 55 """ 56 :param Secondary_html: 57 :return: 一个元组， 58 dir_name --文件夹名称 59 pic_link --- 链接 60 """ 61 ex_link = '<img alt=".*?" src="(.*?)" />' 62 ex_name = '<h1>(.*?)</h1>' # 需要修改的地方 63 pic_link = re.findall(ex_link, Secondary_html, re.S)[0] 64 dir_name = re.findall(ex_name, Secondary_html, re.S)[0] 65 return dir_name, pic_link 66 67 68 # 创建文件夹 69 def create_folder(dir_name): 70 dir_name_cl = "".join(dir_name.split()) 71 dir_name = dir_name_cl 72 if not os.path.exists(dir_name): 73 os.mkdir(dir_name) 74 return dir_name 75 76 77 # 下载图片 78 def down_pic(dir_name, pic_link): 79 """ 80 :param dir_name: 81 :param pic_link: 82 :return: 83 """ 84 img_data = requests.get(url=pic_link, headers=headers).content 85 img_name = pic_link.split('/')[-1] 86 imgPath = dir_name + '/' + img_name 87 with open(imgPath, 'wb') as f: 88 f.write(img_data) 89 return 90 91 92 # 网页主页生成器 93 def create_main_url(url_num): 94 url_ys = '子链接' 95 mian_url_list = [] 96 if url_num > 1: 97 start_num = 2 98 else: 99 start_num = 1 100 101 for url_n in range(start_num, url_num + 1): 102 if url_n != 1: 103 url = url_ys + 'index_%d.html' 104 new_url = format(url % url_n) 105 else: 106 new_url = url_ys 107 mian_url_list.append(new_url) 108 return mian_url_list 109 110 111 # 子网页主页生成器 112 def create_sec_url(url, url_num, *args, **kwargs): 113 """ 114 :param url: 115 :param url_num: 116 :return: 117 """ 118 sec_url_list = [] 119 for url_n in range(1, url_num + 1): 120 if url_n != 1: 121 # new_url = url + '_'+str(url_n)+'.html' 122 begin = url.find("h") 123 end = url.rfind(".") 124 find_url = url[begin:end] 125 new_url = find_url + '_' + str(url_n) + '.html' 126 else: 127 new_url = url 128 sec_url_list.append(new_url) 129 return sec_url_list 130 131 132 # 下载日志生成 133 def create_log(log_content): 134 """ 135 下载日志生成函数 136 :param log_content: 写入log的内容 137 :return: 无 138 """ 139 with open("log.txt", "a") as file: 140 file.write(log_content) 141 return 142 143 144 # 页面记录器 145 def page_record(page_num=0, *args, **kwargs): 146 with open("page_record.txt", "w+") as file: 147 file.write(page_num) 148 return 149 150 151 # 读取配置 152 def page_read(): 153 with open("page_record.txt", "r") as file: 154 r_page_num = file.readline() 155 return r_page_num 156 157 158 ############################################ 爬虫工作区 ############################################ 159 160 n_yema = int(page_read()) 161 # print(n_yema) 162 if polling_page_num > 361: 163 print("您输入的超出轮询范围，请重新输入!") 164 elif polling_page_num > n_yema: 165 end_page_num = polling_page_num 166 print("主程序即将进行") 167 168 # 生成主网页 169 mian_url_list_ys = create_main_url(end_page_num) 170 mian_url_list = mian_url_list_ys[int(n_yema)-1:int(end_page_num)+1] 171 172 173 for url in mian_url_list: 174 n_yema = n_yema + 1 175 sec_url_li = anasy_main_page(download_page(url)[0]) # 分析主链接，获得套图列表 176 print(len(sec_url_li), sec_url_li) 177 log_mian_start = "*" * 15 + "第" + str(n_yema) + "页，开始下载-->" + url + "*" * 15 178 print(log_mian_start) # 某些页面开始下载的提示 179 n_tao = 0 180 for url_sec in sec_url_li[0:31]: 181 n_tao = n_tao + 1 182 dir_name = anasy_Secondary_page(download_page(url_sec, html_encode="utf-8")[0])[0] # 分析指定的套图链接，获得套图的名称 183 184 185 print("*" * 15 + "第" + str(n_yema) + "页，第" + str(n_tao) + "套图--" + dir_name + "--" + "开始下载" + "*" * 15) 186 dir_name_sj = create_folder(dir_name) 187 sec_url_list = create_sec_url(url_sec, 60) 188 m = 0 189 for pic_link in sec_url_list: 190 m = m + 1 191 page_text, response_status_pic = download_page(pic_link) 192 if response_status_pic == 200: 193 donw_pic_link = anasy_Secondary_page(page_text)[1] # 分析指定的套图链接，获得图片链接 194 down_pic(dir_name_sj, donw_pic_link) 195 print("第" + str(m) + "张图片下成功", donw_pic_link) 196 time.sleep(1) 197 else: 198 continue 199 200 print("第" + str(n_yema) + "页，第" + str(n_tao) + "套图--" + dir_name + "全部图片下载完毕" + " ") 201 log_text = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) + "---" + "第" + str( 202 n_yema) + "页，第" + str(n_tao) + "套图--" + dir_name + "已经下载完毕" + " " 203 create_log(log_content=log_text) 204 log_main_end = "*" * 10 + "第" + str(n_yema) + "页，下载完成-->" + url + "*" * 10 + " " 205 print(log_main_end) 206 # 要记录已经下载的主页的页面页码（n_yema） 207 page_record(str(n_yema))
