爬取个人随笔内容——练手，待补充

import requests,lxml
from bs4 import BeautifulSoup
url='https://www.cnblogs.com/wjlv/default.html?page=2'  # 打开网页
html_index = requests.get(url).text  # 获取请求内容
soap = BeautifulSoup(html_index,"lxml")
a_list = soap.find_all('a',{"class":"postTitle2"})  # 得到所有随笔标题和地址标签
for h in a_list:
    soap = BeautifulSoup(str(h),'lxml')
    # print('{}:{}'.format(soap.find('a').text,soap.find('a')['href']))  # 得到单个随笔的地址
    article = requests.get(soap.find('a')['href']).text  # 得到单个随笔的内容
    soap_a = BeautifulSoup(article,'lxml')
    p_lab = soap_a.find_all('p')  # 获取单个随笔内容标签
    for txt in p_lab:
        soap_t = BeautifulSoup(str(txt),'lxml')
        print(soap_t.find('p').text)  # 得到所有标签的具体内容
拿书旗小树网站的一片小说练练手：
还是有很多地方需要优化：
1、重复提取内容的步骤较多
2、提取效率较为低下，多线程
3、小说获取较单一、固定

import lxml, requests, os
from bs4 import BeautifulSoup

url = 'http://book.txtbook.com.cn/shu/6478/chapterlist.html'
response = requests.get(url).text  # 请求小说章节总览页
soap = BeautifulSoup(response, 'lxml')
title_list = soap.find('div', {"class": "t_list6"})
soap_t = BeautifulSoup(str(title_list), 'lxml')
href_list = soap_t.find_all('a')
for href in href_list:
    soap_h = BeautifulSoup(str(href), 'lxml')
    content_url, content_title = soap_h.find('a', {'class': 'nocur'})['href'], soap_h.find('a').text  # 获取章节名称和url
    content_response = requests.get(content_url).text
    soap_r = BeautifulSoup(content_response, 'lxml')
    content_p = soap_r.find_all('div', {'id': 'chaptercontent'})  # 多属性确定内容位置
    for content in content_p:
        soap_c = BeautifulSoup(str(content), 'lxml')
        # print(soap_c.find('p'))
        if not os.path.exists(r"E:python_project1dayooks"):  # 创建存放目录
            os.makedirs(r'../books/')
        with open(r'../books/' + content_title + '.txt', 'w+',encoding='utf8') as f:
            f.writelines(soap_c.find('p').text.split("<br/><br/>"))  # 写入章节内容
        print('%s has already download' %content_title)

爬取个人博客：

import re, os, requests


def get_url(url, pattern=None):
    try:
        response = requests.get(url)
        section_info = []
        if response.status_code == 200:
            if pattern is not None:
                p_section = re.compile(pattern, re.S)  # r'class="postTitle2[^.]*?ef="(.*?)">(.*?)</a>'
                section_info = re.findall(p_section, response.text)
            return section_info
        else:
            return None
    except Exception as e:
        return e

def write_content(content,section):
    if not os.path.exists(r'../blogs/'):
        os.makedirs(r'../blogs/')
    try:
        with open(r'../blogs/'+ section.strip().replace('.','_').replace('——','_').replace(' ','') + '.txt','w+') as f:
            f.write(str(content).strip(']['))
            print('%s 下载完成……' %section)
    except IOError:
        print('%s 下载失败……' %section)

def get_content(section_list, pattern):
    for section in section_list:
        content_list = get_url(section[0], pattern)
        content = []
        for p in content_list:
            p = p.strip()
            content.append(p)
        write_content(content,section[1])


if __name__ == '__main__':
    url = 'https://www.cnblogs.com/wjlv/'
    section_list = get_url(url, r'class="postTitle2[^.]*?ef="(.*?)">(.*?)</a>')
    get_content(section_list, r'<p>(.*?)</p>')