爬取7160

优化后的代码如下,

先用循环创建20个目录,然后循环写入这20个目录,每个最多写入50000

#coding=utf-8
import os
import random
import sys


import urllib.request
from bs4 import BeautifulSoup
from urllib import error
import re
ls = ['meinv','zhenrenxiu',"lianglichemo",'rentiyishu','xiaohua','lianglichemo']
file_list = os.listdir("d:\craw\")

def validateTitle(title):
   rstr = r"[/\:*?"<>|]"  # '/  : * ? " < > |'
   new_title = re.sub(rstr, "_", title)  # 替换为下划线
   return new_title

def get_file_name():
   file = random.sample(file_list,1)[0]
   path = 'd:\craw/'+ str(file);
   if  os.path.isdir(path):
      total_num = len(os.listdir('d:\craw\'+ str(file)))
      if total_num >= 50000:
         file = get_file_name()
   else:
      os.mkdir(path)
      print("创建目录"+ str(path))

   return str(path)+'\'
for k in ls:
   for j in range(1,101111):
      url_origin = "http://www.7160.com/"+str(k)+"/"+str(j)
      print(url_origin)
      try:
         page_obj = urllib.request.urlopen(url_origin)
         page_soup = BeautifulSoup(page_obj,'lxml')
         total_page_obj = page_soup.find(text=re.compile('共')).string
         pattern = re.compile(r'd+')
         match = pattern.search(total_page_obj)

         if match == None:
            total_page = 0;
         else:
            total_page = match.group();

         for i in range(1,int(total_page)+1):
            if i == 1 :
               url = url_origin+"/index.html"
            else:
               url = url_origin+"/index_"+str(i)+".html"
            request = urllib.request.Request(url)
            try:
               res = urllib.request.urlopen(request)

               soup = BeautifulSoup(res,'lxml')
               title_obj = soup.find(attrs={"class":"picmainer"})

               if title_obj is not None:
                  print(url)
                  title = title_obj.h1.string
                  content = soup.find('img')
                  src = content.get("src")
                  file_name = validateTitle(title)+".jpg"
                  urllib.request.urlretrieve(src, str(get_file_name())+file_name)
                  print(str(get_file_name())+file_name+"保存成功")
            except Exception  as e:
               print("异常"+str(e))
      except Exception  as e:
               print("异常"+str(e))