《python网络数据采集》读后感 第五章：存储数据

1.媒体文件：存储媒体文件有两种主要方式：a.获取URL链接。b.将源文件下载下来。

import os
from urllib.request import urlretrieve
from urllib.request import urlopen
from bs4 import BeautifulSoup

downloadDirectory = "downloaded"
baseUrl = "http://pythonscraping.com"

def getAbsoluteURL(baseUrl, source):
    if source.startswith("http://www."):
        url = "http://"+source[11:]

    elif source.startswith("http://"):
        url = source

    elif source.startswith("www."):
        url = source[4:]
        url = "http://" + source

    else:
        url = baseUrl + "/" + source
    if baseUrl not in url:
        return None

    return url

def getDownloadPath(baseUrl, absoluteUrl, downloadDirectory):
    path = absoluteUrl.replace("www.", "")

    path = path.replace(baseUrl, "")
    path = downloadDirectory + path

    directory = os.path.dirname(path)


    if not os.path.exists(directory):
        os.makedirs(directory)

    return path

html = urlopen("http://www.pythonscraping.com")
bsObj = BeautifulSoup(html)
downloadList = bsObj.findAll(src=True)

for download in downloadList:
    fileUrl = getAbsoluteURL(baseUrl, download["src"])
    if fileUrl is not None:
        print(fileUrl)
urlretrieve(fileUrl, getDownloadPath(baseUrl, fileUrl, downloadDirectory))