《python网络数据采集》读后感 第三章：开始采集

dui
 1 from urllib.request import urlopen
from bs4 import BeautifulSoup
import re
import random
import datetime

pages = set()
random.seed(datetime.datetime.now())

#获取当前页面所有内链的表
def getInternalLinks(bsObj,includeUrl):
    internalLinks = []
    #找出所有以"/"开头的链接
    for link in bsObj.findAll("a",href = re.compile("^(/|.*"+includeUrl+")")):
        if link.attrs['href'] is not None:
            if link.attrs['href'] not in internalLinks:
                internalLinks.append(link.attrs['href'])

    return internalLinks

#获取当前页面所有外链的表
def getExternalLinks(bsObj,excludeUrl):
    externalLinks = []
    #找出所有以'http'和'www'开头且不包含当前URL的链接
    for link in bsObj.findAll("a",
                              href=re.compile("^(http|www)((?!"+excludeUrl+").)*$")):
        if link.attrs['href'] is not None:
            if link.attrs['href'] not in externalLinks:
                externalLinks.append(link.attrs['href'])

    return externalLinks

def splitAddress(address):
    addressParts = address.replace("http://","").split("/")
    return addressParts


def getNextExternalLink(param):
    pass


def getRandomExternalLink(startingPage):
    html = urlopen(startingPage)
    bsObj = BeautifulSoup(html)
    externalLinks = getExternalLinks(bsObj, splitAddress(startingPage)[0])
    if len(externalLinks) == 0:
        internalLinks = getInternalLinks(startingPage)
        return getNextExternalLink(internalLinks[random.randint(0,
                                    len(internalLinks)-1)])
    else:
        return externalLinks[random.randint(0, len(externalLinks)-1)]

def followExternalOnly(startingSite):
    externalLink = getRandomExternalLink("http://oreilly.com")
    print("随机外链是:"+externalLink)
    followExternalOnly(externalLink)

followExternalOnly("http://oreilly.com")

上面这个程序从 http://oreilly.com 开始，然后随机地从一个外链跳到另一个外链。

网站首页上并不能保证一直能发现外链。这时为了能够发现外链，递归地深入一个网站直到找到一个外链才停止。