第二次作业

作业一

（1）在中国气象网（http://www.weather.com.cn）给定城市集的7日天气预报，并保存在数据库。

思路：

           1.首先建立一个类，用于实现建立数据库，并将数据写入数据库

           2.然后再建立一个用于实现爬取数据的类

实现代码：

# 031804127wl

from bs4 import BeautifulSoup
from bs4 import UnicodeDammit
import urllib.request
import sqlite3


class WeatherDB:
    def openDB(self):
        self.con = sqlite3.connect("weathers.db")
        self.cursor = self.con.cursor()
        try:
            self.cursor.execute("create table weathers (wCity varchar(16), wDate varchar(16), wWeather varchar(64), "
                                "wTemp varchar(32), constraint pk_weather primary key(wCity, wDate))")
        except:
            self.cursor.execute("delete from weathers")

    def closeDB(self):
        self.con.commit()
        self.con.close()

    def insert(self, city, date, weather, temp):
        try:
            self.cursor.execute("insert into weathers (wCity, wDate, wWeather, wTemp)values(?, ?, ?, ?)",
                                (city, date, weather, temp))
        except Exception as err:
            print(err)

    def show(self):
        self.cursor.execute("select * form weathers")
        rows = self.cursor.fetchall()
        print("%-16s%-16s%-32s%-16s" % ("city", "date", "weather", "temp"))
        for row in rows:
            print("%-16s%-16s%-32s%-16s" % (row[0], row[1], row[2], row[3]))


class WeatherForecast:
    def __init__(self):
        self.headers = {"User-Agent": "Mozilla/5.0"}
        self.cityCode = {"北京": "101010100", "上海": "101020100", "广州": "101280101", "深圳": "101280601"}

    def forecastCity(self, city):
        if city not in self.cityCode.keys():
            print(city + " code cannot be found")
            return

        url = "http://www.weather.com.cn/weather/" + self.cityCode[city] + ".shtml"
        try:
            req = urllib.request.Request(url, headers=self.headers)
            data = urllib.request.urlopen(req)
            data = data.read()
            dammit = UnicodeDammit(data, ["utf-8", "gbk"])
            data = dammit.unicode_markup
            soup = BeautifulSoup(data, "lxml")
            lis = soup.select("ul[class='t clearfix'] li")
            for li in lis:
                try:
                    date = li.select('h1')[0].text
                    weather = li.select('p[class="wea"]')[0].text
                    temp = li.select('p[class="tem"] span')[0].text + "/" + li.select('p[class="tem"] i')[0].text
                    print(city, date, weather, temp)
                except Exception as err:
                    print(err)
        except Exception as err:
            print(err)

    def process(self, cities):
        self.db = WeatherDB()
        self.db.openDB()

        for city in cities:
            self.forecastCity(city)

        # self.db.show()
        self.db.closeDB()


ws = WeatherForecast()
ws.process(["北京", "上海", "广州", "深圳"])
print("completed")

结果图片：

 （2）实验心得：

  学习了如何建立数据库并向其中存入数据，也算是对上个学期学过的一些内容的复习。

作业二：

（1）用requests和BeautifulSoup库方法定向爬取股票相关信息。

思路：

        1.利用浏览器自带的抓包工具，查找股票列表加载使用的url

        2.分析得到的数据，进行爬取

        3.规范输出格式

实现代码：

# 031804127wl

import re
import requests
import json
import pandas as pd

# 显示所有列
pd.set_option('display.max_columns', None)
# 显示所有行
pd.set_option('display.max_rows', None)
# 列名与数据对其显示
pd.set_option('display.unicode.ambiguous_as_wide', True)
pd.set_option('display.unicode.east_asian_width', True)
# 横向展示5000个字符
pd.set_option('display.width', 5000)


def HTML(url):
    gupiao_list = []
    headers = {
        "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) "
                      "Chrome/80.0.3987.149 Safari/537.36"}
    try:
        r = requests.get(url, headers=headers, timeout=30)
        r.raise_for_status()
        r.encoding = r.apparent_encoding
        html = r.text
    except Exception as e:
        print("wrong:" + e)
    # 使用正则表达式，爬取data下的所有数据
    pat = re.compile("[{.*?}]")
    data = pat.findall(html)
    js = json.loads(data[0])  # 将json代码规范化
    gupiao_list.append(("代码", "名称", "最新价", "涨跌幅", "涨跌额", "成交量", "成交额", "振幅", "最高", "最低", "今开", "昨收", "量比"))
    for i in range(len(js)):
        diezhangfu = str(js[i]["f3"]) + "%"
        zhenfu = str(js[i]["f7"]) + "%"
        gupiao_list.append((js[i]["f12"], js[i]["f14"], js[i]["f2"], diezhangfu, js[i]["f4"], js[i]["f5"], js[i]["f6"],
                            zhenfu, js[i]["f15"], js[i]["f16"], js[i]["f17"], js[i]["f18"], js[i]["f10"]))
    # 将列表信息规范格式
    df = pd.DataFrame(gupiao_list)
    return df

def main():
    # 爬取前两页的内容
    for i in range(1, 3):
        url = "http://28.push2.eastmoney.com/api/qt/clist/get?cb=jQuery11240023072405517482908_1601430405294&pn=" + str(
            i) + "&pz" 
                 "=20&po=1&np=1&ut=bd1d9ddb04089700cf9c27f6f7426281&fltt=2&invt=2&fid=f3&fs=m:0+t:6,m:0+t:13,m:0+t:80," 
                 "m:1+t:2,m:1+t:23&fields=f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f12,f13,f14,f15,f16,f17,f18,f20,f21,f23,f24," 
                 "f25,f22,f11,f62,f128,f136,f115,f152&_=1601430405304 "
        print(HTML(url))


main()

结果图片：

（2）实验心得

  学会了使用抓包工具获取网页数据，初步了解了一下json库的使用方法。并且由于最后输出的数据量比较大，我使用了pandas库对数据的输出进行了规范，又掌握了一个使输出更加美观的方法。

作业三

（1）根据自选3位数+学号后3位选取股票，获取印股票信息。

思路：

       我的学号后三位数是127，因此直接输出代码后缀是127的股票信息就可以了，由于这样的股票只是少数，因此不需要使用pandas库，直接手动规范输出格式。由于股票的位置随时在变动，因此当发现没有数据输出的时候，改变页码范围就可以了。

实现代码：

# 031804127wl

import re
import requests
import json

def HTML(url):
    headers = {
        "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) "
                      "Chrome/80.0.3987.149 Safari/537.36"}
    try:
        r = requests.get(url, headers=headers, timeout=30)
        r.raise_for_status()
        r.encoding = r.apparent_encoding
        html = r.text
    except Exception as e:
        print("wrong:" + e)
    # 使用正则表达式，爬取data下的所有数据
    pat = re.compile("[{.*?}]")
    data = pat.findall(html)
    js = json.loads(data[0])  # 将json代码规范化
    for i in range(len(js)):
        diezhangfu = str(js[i]["f3"]) + "%"
        zhenfu = str(js[i]["f7"]) + "%"
        daima = js[i]["f12"]
        # 输出代码后缀为127的股票数据
        if daima[3:] == '127':
            print("代码:" + str(js[i]["f12"]) + "	" + "名称:" + str(js[i]["f14"]) + "	" + "最新价:" + str(js[i]["f2"])
                  + "	" + "涨跌幅:" + str(diezhangfu) + "	" + "涨跌额:" + str(js[i]["f4"]) + "	" + "成交量:" + str(
                js[i]["f5"])
                  + "	" + "成交额:" + str(js[i]["f6"]) + "	" + "振幅:" + str(zhenfu) + "	" + "最高:" + str(
                js[i]["f15"]) + "	" +
                  "最低:" + str(js[i]["f16"]) + "	" + "今开:" + str(js[i]["f17"]) + "	 " + "昨收:" + str(js[i]["f18"]) +
                  "	" + "量比:" + str(js[i]["f10"]))
        else:
            continue


def main():
    # 爬取30-40页的内容
    for i in range(30, 41):
        url = "http://28.push2.eastmoney.com/api/qt/clist/get?cb=jQuery11240023072405517482908_1601430405294&pn=" + str(
            i) + "&pz" 
                 "=20&po=1&np=1&ut=bd1d9ddb04089700cf9c27f6f7426281&fltt=2&invt=2&fid=f3&fs=m:0+t:6,m:0+t:13,m:0+t:80," 
                 "m:1+t:2,m:1+t:23&fields=f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f12,f13,f14,f15,f16,f17,f18,f20,f21,f23,f24," 
                 "f25,f22,f11,f62,f128,f136,f115,f152&_=1601430405304 "
        HTML(url)


main()

 结果图片：

（2）实验心得

  就是在前一个作业的基础上对输出进行一些修改。