第一部分-创建爬虫
1.urllib
1)urllib.request
request.urlopen(url)
request.urlretrieve 可以根据文件的 URL 下载文件
2)urllib.parse
3)urllib.error
2.异常：
try...except...else...
常见异常：HTTPError,AttributeError,
3.BeautifulSoup
from bs4 import BeautifulSoup
bsObj=BeautifulSoup(html,'lxml')
1)
print(bsObj.text)
print(bsObj.html)
print(bsObj.p.a)
...
2)
findAll(tag, attributes, recursive, text, limit, keywords) #返回一个ResultSet
find(tag, attributes, recursive, text, keywords) #返回一个Tag
借助它们,你可以通过标签的不同属性轻松地过滤 HTML 页面，查找需要的标签组或单个标签
例：
.findAll({"h1","h2","h3","h4","h5","h6"})
.findAll("span", {"class":{"green", "red"}})
.findAll(id="text") 同 .findAll("", {"id":"text"})
.findAll(src=True) 有src属性的标签
3)
get_text() 会把你正在处理的 HTML 文档中所有的标签都清除，然后返回一个只包含文字的str
4)返回类型NavigatorString
.children （所有子标签）
.next_sibling（ 下一个兄弟标签 ）.next_siblings（所有之后的兄弟标签）
.previous_sibling（上一个兄弟标签）.previous（所有之前的兄弟标签）
.parent （直接父标签 ）.parents（所有父标签）、
5）
.attrs 获取标签所有属性(dict)
.attrs['src'] 获取src值
6)正则表达式
7)lambda表达式
#获取有两个属性的标签：
bsObj.findAll(lambda tag: len(tag.attrs) == 2)
4.Scrapy
//TODO
5.JSON
把 JSON 转换成字典，
JSON 数组转换成列表，
JSON 字符串转换成 Python 字符串。
常用函数：loads,get
6.存储数据
1)下载
from urllib.request import urlretrieve
urlretrieve(resourceLocation,fileName)
2）CSV（Comma-Separated Values）
import csv
csvFile=open("test.csv","w+")
try:
writer=csv.writer(csvFile)
writer.writerow(('青山隐隐水迢迢 秋尽江南草未凋','24桥明月夜'))
for i in range(1,5):
writer.writerow((i,i+2,i*2))
finally:
csvFile.close()
3）MySQL
import pymysql
#获取连接 获取光标
conn=pymysql.connect(host='localhost',user='root',passwd=None)
cur=conn.cursor()
#执行SQL语句
cur.execute('use ssm01')
cur.execute('select * from user')
print(cur.fetchone())#获取一条数据
#关闭资源
cur.close()
coon.close()
4）Email
//TODO
7.读取文档
1)读取txt
from urllib.request import urlopen
txt=urlopen('http://www.pythonscraping.com/pages/warandpeace/chapter1.txt')
print(txt.read())
2)读取csv
#从网上直接把文件读成一个字符串，然后转换成一个 StringIO 对象，使它具有文件的属性。
from urllib.request import urlopen
from io import StringIO
import csv
data = urlopen('http://pythonscraping.com/files/MontyPythonAlbums.csv').read().decode('utf-8')
dataFile=StringIO(data)
csvFile=csv.reader(dataFile)
for row in csvFile:
print(row)
3）读取PDF
#PDFMiner3K
#把任意 PDF 读成字符串，然后用 StringIO 转换成文件对象
from urllib.request import urlopen
from pdfminer.pdfinterp import PDFResourceManager, process_pdf
from pdfminer.converter import TextConverter
from pdfminer.layout import LAParams
from io import StringIO
def readPDF(pdfFile):
rsrcmgr = PDFResourceManager()
retstr = StringIO()
laparams = LAParams()
device = TextConverter(rsrcmgr, retstr, laparams=laparams)
process_pdf(rsrcmgr, device, pdfFile)
device.close()
content = retstr.getvalue()
retstr.close()
return content
pdfFile = urlopen("http://pythonscraping.com/pages/warandpeace/chapter1.pdf")
outputString = readPDF(pdfFile)
print(outputString)
pdfFile.close()
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
例
3-1.网络数据采集
#从 http://oreilly.com 开始，然后随机地从一个外链跳到另一个外链。
from urllib.request import urlopen
from bs4 import BeautifulSoup
import re
import datetime
import random
pages = set()
random.seed(datetime.datetime.now())
# 获取页面所有内链的列表
def getInternalLinks(bsObj, includeUrl):
internalLinks = []
# 找出所有以"/"开头的链接
for link in bsObj.findAll("a", href=re.compile("^(/|.*"+includeUrl+")")):
if link.attrs['href'] is not None:
if link.attrs['href'] not in internalLinks:
internalLinks.append(link.attrs['href'])
return internalLinks
# 获取页面所有外链的列表
def getExternalLinks(bsObj, excludeUrl):
externalLinks = []
# 找出所有以"http"或"www"开头且不包含当前URL的链接
for link in bsObj.findAll("a",href=re.compile("^(http|www)((?!"+excludeUrl+").)*$")):
if link.attrs['href'] is not None:
if link.attrs['href'] not in externalLinks:
externalLinks.append(link.attrs['href'])
return externalLinks
def splitAddress(address):
addressParts = address.replace("http://", "").split("/")
return addressParts
def getRandomExternalLink(startingPage):
html = urlopen(startingPage)
bsObj = BeautifulSoup(html,'lxml')
externalLinks = getExternalLinks(bsObj, splitAddress(startingPage)[0])
if len(externalLinks) == 0:
internalLinks = getInternalLinks(startingPage)
return getNextExternalLink(internalLinks[random.randint(0,len(internalLinks)-1)])
else:
return externalLinks[random.randint(0, len(externalLinks)-1)]
def followExternalOnly(startingSite):
externalLink = getRandomExternalLink("http://oreilly.com")
print("随机外链是："+externalLink)
followExternalOnly(externalLink)
followExternalOnly("http://oreilly.com")
5-1.JSON
import json
jsonString='{
"arrayOfNums":[{"number":0},{"number":1},{"number":2}],
"arrayOfFruits":[{"fruit":"apple"},{"fruit":"banana"},{"fruit":"pear"}]
}'
jsonObj=json.loads(jsonString)
print(jsonObj.get("arrayOfFruits")[2].get("fruit"))
6-1.把 http://pythonscraping.com 的所有图片下载下来
from urllib.request import urlretrieve
from urllib.request import urlopen
from bs4 import BeautifulSoup
def pageSrc(url):
html=urlopen(url)
bsObj=BeautifulSoup(html,'lxml')
srcList=bsObj.findAll("img",src=True)
urlList=[]
for i in srcList:
urlList.append(i['src'])
return urlList
def getInternalLinks(bsObj,includeUrl):
internalLinks = []
# 找出所有以"/"开头的链接
for link in bsObj.findAll("a", href=re.compile("^(/|.*"+includeUrl+")")):
if link.attrs['href'] is not None:
if link.attrs['href'] not in internalLinks:
internalLinks.append(link.attrs['href'])
return internalLinks
def allimgs(url):
#找到该页面所有的img src
srcset=set()
for i in pageSrc(url):
if i not in srcset:
print(i)
srcset.add(i)
name=i.split('/').pop()
urlretrieve(i,name)
#找到该页面的所有内链
html=urlopen(url)
bsObj=BeautifulSoup(html,'lxml')
for i in getInternalLinks(bsObj,url):
newUrl=url+i
for j in pageSrc(newUrl):
if j not in srcset:
srcset.add(i)
print(j)
name=j.split('/').pop()
urlretrieve(j,name)
url="http://pythonscraping.com"
allimgs(url)
6-2.存储到CSV
#获取 HTML 表格并写入 CSV 文件
import csv
from urllib.request import urlopen
from bs4 import BeautifulSoup
html = urlopen("http://en.wikipedia.org/wiki/Comparison_of_text_editors")
bsObj = BeautifulSoup(html,'lxml')
# 主对比表格是当前页面上的第一个表格
table = bsObj.findAll("table",{"class":"wikitable"})[0]
rows = table.findAll("tr")
csvFile = open("editors.csv", 'wt', newline='',encoding='utf-8')
writer = csv.writer(csvFile)
try:
for row in rows:
csvRow = []
for cell in row.findAll(['td', 'th']):
csvRow.append(cell.get_text()[:-1])
print(csvRow)
writer.writerow(csvRow)
finally:
csvFile.close()
6-3.存储到mysql
#存储维基百科数据
from urllib.request import urlopen
from bs4 import BeautifulSoup
import re
import datetime
import random
import pymysql
conn = pymysql.connect(host='127.0.0.1',user='root', passwd=None, charset='utf8')
cur = conn.cursor()
cur.execute("USE ssm01")
cur.execute("CREATE TABLE pages(title varchar(200),content varchar(3000))")
random.seed(datetime.datetime.now())
#存储到数据库
def store(title, content):
cur.execute("INSERT INTO pages (title, content) VALUES ("%s","%s")", (title, content))
cur.connection.commit()
#找到数据 存储到数据库
def getLinks(articleUrl):
html = urlopen("http://en.wikipedia.org"+articleUrl)
bsObj = BeautifulSoup(html,'lxml')
title = bsObj.find("h1").get_text()
content = bsObj.find("div", {"id":"mw-content-text"}).find("p").get_text()
store(title, content)
return bsObj.find("div", {"id":"bodyContent"}).findAll("a",href=re.compile("^(/wiki/)((?!:).)*$"))
links = getLinks("/wiki/Kevin_Bacon")
try:
while len(links) > 0:
newArticle = links[random.randint(0, len(links)-1)].attrs["href"]
print(newArticle)
links = getLinks(newArticle)
finally:
cur.close()
conn.close()