Python爬虫：使用正则表达式爬取网站电影信息

以爬取电影天堂喜剧片前5页信息为例，代码如下：

# coding:UTF-8

import requests
import re

def mov():
    headers={'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36',
         'Host':'www.dy2018.com'}
    #url=('https://www.dy2018.com/1/')
    r = requests.get(url,headers=headers)
    data = r.text.encode("latin1").decode("gbk").replace(u'u3000', u'')
    pattern = re.compile('<a.*?class=ulink.*?</a>.*?<a.*?href="(.*?)" class="ulink".*?title=.*?>(.*?)</a>.*?</td>.*?<td.*?style="padding-left.*?>.*?<font.*?>.*?日期：(.*?)</font>.*?<fontscolor=.*?◎评分:s(.*?)s.*?</font>.*?<td.*?colspan="2".*?style=.*?<p>◎片名:(.*?)s.*?◎别名:(.*?)s.*?导演:(.*?)s.*?</p>.*?<p>s.*?类型:(.*?)s.*?<p>.*?主演:(.*?)</p>',re.S)
    #pattern = re.compile('<fontscolor=":(.*?)</font>.*?<tdscolspan="2".*?<p>(.*?) </p>.*?<p>(.*?)</p>.*?<p>(.*?)</p>.*?</td>.*?</tr>',re.S)
    items = re.findall(pattern,data)
    for item in items:
        yield{
            'href':item[0],
            '标题':item[1],
            '日期':item[2],
            '评分':item[3],
            '片名':item[4],
            '别名':item[5],
            '导演':item[6],
            '类型':item[7],
            '主演':item[8]
        }

def save_file(d):
        with open('t1.html','a',encoding='utf-8') as f:
            f.write('
第'+str(i)+'页
')
            for m in d:
                f.write(str(m)+'
')
34 
for i in range(1,6):
    d=mov()
    if i==1:
        url='https://www.dy2018.com/1/'
        save_file(d)
        print('第 1 页爬取完成！')
    else:
        url = ('https://www.dy2018.com/1/index_' + str(i) + '.html')
        save_file(d)
        print('第',i,'页爬取完成！')

电影信息爬取效果：

爬取下载地址代码如下：

# coding:UTF-8

import requests
import re

def hrefs():
    headers={'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36',
         'Host':'www.dy2018.com'}
    #url=('https://www.dy2018.com/1/')
    r = requests.get(url,headers=headers)
    pattern = re.compile('<a.*?class=ulink.*?</a>.*?<a.*?href="(.*?)" class="ulink"',re.S)
    href = re.findall(pattern,r.text)
    return href

def inf(link):
    for h in link:
        durl= ('https://www.dy2018.com'+ h)
        headers={'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36',
             'Host':'www.dy2018.com'}
        dr = requests.get(durl,headers=headers)
        ddata = dr.text.encode("latin1").decode("gbk").replace(u'u3000', u'')
        pattern = re.compile('alt=.*?译.*?名(.*?)</p>.*?片.*?名(.*?)</p>.*?style="margin.*?href="(.*?)">.*?',re.S)
        info = re.findall(pattern,ddata)
        for item in info:
            yield{
                #'译名':item[0],
                '片名':item[1],
                '下载地址':item[2]
        }

def save_file(link):
        with open('t2.html','a',encoding='utf-8') as f:
            #link=hrefs()
            f.write('
第'+str(i)+'页
')
            for ins in inf(link):
                f.write(str(ins)+'
')
38 
for i in range(1,6):
    if i==1:
        url='https://www.dy2018.com/1/'
        link=hrefs()
        save_file(link)
        print('第 1 页爬取完成！')
    else:
        url = ('https://www.dy2018.com/1/index_' + str(i) + '.html')
        link=hrefs()
        save_file(link)
        print('第',i,'页爬取完成！')

爬取下载地址效果如下：