url请求，request请求，解析库beauitfulsoup，解析库lxml

url请求

from urllib.request import urlopen
url="****"
respones = urlopen(url)
content =  respones.read()
content = content.decode('utf-8')
print(content)

request请求

import requests
url="***"
headers = {'Accept': '*/*',
               'Accept-Language': 'en-US,en;q=0.8',
               'Cache-Control': 'max-age=0',
               'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36',
               'Connection': 'keep-alive',
               'Referer': 'http://www.baidu.com/'
               }
res = requests.get (url,headers=headers)#加headers头是为了伪装成浏览器浏览网页
print(res.status_code)#打印状态码
print(res.text)#打印文本
print(res.content)#打印图片或者视频文本都可以

解析库BeautifulSoup

import requests
from bs4 import BeautifulSoup
url="http://news.qq.com/"
headers = {'Accept': '*/*',
               'Accept-Language': 'en-US,en;q=0.8',
               'Cache-Control': 'max-age=0',
               'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36',
               'Connection': 'keep-alive',
               'Referer': 'http://www.baidu.com/'
               }
res = requests.get (url,headers=headers)#加headers头是为了伪装成浏览器浏览网页
Soup = BeautifulSoup(res.text.encode("utf-8"),'lxml')
en = Soup.find_all('en',attrs={'class':'f14 124'})
for i in en:
    title = i.a.get_text()
    link = i.a['href']
    print({
        '标题':title,
        '链接':link
    })

解析库lxml，xpath表达法

import requests
from lxml import etree
url="http://news.qq.com/"
headers = {'Accept': '*/*',
               'Accept-Language': 'en-US,en;q=0.8',
               'Cache-Control': 'max-age=0',
               'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36',
               'Connection': 'keep-alive',
               'Referer': 'http://www.baidu.com/'
               }
html = requests.get (url,headers=headers)#加headers头是为了伪装成浏览器浏览网页
con  = etree.HTML(html.text)
title = con.xpath('//en[@class="f14 124"]/a/text()')
link = con.xpath('//en[@class="f14 124"]/a/@href')
for i in zip(title,link):
    print({
        '标题':i[0],
        '链接':i[1]
    })

selesct方法

import requests
from bs4 import BeautifulSoup
url="http://news.qq.com/"
headers = {'Accept': '*/*',
               'Accept-Language': 'en-US,en;q=0.8',
               'Cache-Control': 'max-age=0',
               'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36',
               'Connection': 'keep-alive',
               'Referer': 'http://www.baidu.com/'
               }
res = requests.get (url,headers=headers)#加headers头是为了伪装成浏览器浏览网页
Soup = BeautifulSoup(res.text.encode("utf-8"),'lxml')
en = Soup.select('en[class="f12 124"] a')
for i in en:
    title = i.a.get_text()
    link = i.a['href']
    print({
        '标题':i[0],
        '链接':i[1]
    })