最基础网页爬虫

第一个网页文本爬虫程序（没有添加下载器）：

import requests
from bs4 import BeautifulSoup
import os

headers={'User-Agent':"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTMl,like Gecko) Chrome/22.0.1207.1 Safari/537.1"}
url_begin= 'http://www.liaoxuefeng.com/wiki/0014316089557264a6b348958f449949df42a6d3a2e542c000'
start_url=requests.get(url_begin,headers=headers)
#print(start_url.text)
Soup=BeautifulSoup(start_url.text,'lxml')
link_list=Soup.find('div',class_="x-sidebar-left-content").find_all('a')
#print(link_list)

for link in link_list:
        url='http://www.liaoxuefeng.com' + link['href']
        html=requests.get(url,headers=headers)
        html_Soup=BeautifulSoup(html.text,'lxml')
        title_list = html_Soup.find('div', class_="x-content").find_all('h4')
        # print(title_list)
        for title in title_list:
            titlereal = title.get_text()
            print(titlereal)

        content_list = html_Soup.find("div", class_="x-wiki-content").find_all('p')
        for content in content_list:
            # print(content)
            contentreal = content.get_text()
            print(contentreal)

第二个网页图片爬虫（引入os模块，可以将网页内容爬取到本地文件夹）

import requests
from bs4 import BeautifulSoup
import os
import urllib
import urllib3

url= 'http://www.dbmeinv.com/?pager_oofset=1'
x=0

def crawl(url):
    headers = {
        'User-Agent': "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTMl,like Gecko) Chrome/22.0.1207.1 Safari/537.1"}
    req=requests.get(url,headers=headers)

    Soup=BeautifulSoup(req.text,'lxml')
    link_list=Soup.find_all('img')
    for girl in link_list:
        link= girl.get('src')
        print(link)

        global x
        path=r'/Users/wangxitao/Desktop/douban'
        local=os.path.join(path,'image\%s.jpg'%x)
        urllib.request.urlretrieve(link,local)
                                         #'image\%s.jpg'%x

        x+=1
        print("正在下载第%s张"%x)

for page in range(1,10):
    page+=1
    url='http://www.dbmeinv.com/?pager_offset=%d'%page
    crawl(url)

print('爬取完毕')