《鲜活的数据-第2章 处理数据》有关代码

2.1.3 自动收集数据

import urllib2

page = urllib2.urlopen("https://www.wunderground.com/history/airport/ZHCC/2017/9/8/DailyHistory.html")

from BeautifulSoup import BeautifulSoup

soup = BeautifulSoup(page)

images = soup.findAll('img')

first_image = images[0]

print first_image

wxvalue = soup.findAll(attrs={"class":"wx-value"})

print wxvalue

print wxvalue[0]

print wxvalue[0].span.string #AttributeError: 'NoneType' object has no attribute 'string'

print wxvalue[0].contents[0].string

for m in range(1, 13):
    for d in range(1, 32):
 
      # Check if already gone through month
      if (m == 2 and d > 28):
        break
      elif (m in [4, 6, 9, 11] and d > 30):
        break
 
      # Open wunderground.com url
      timestamp = '2016' + str(m) + str(d)
      print "Getting data for " + timestamp
      #url = "http://www.wunderground.com/history/airport/KBUF/2009/" + str(m) + "/" + str(d) + "/DailyHistory.html"
      url = "https://www.wunderground.com/history/airport/ZHCC/2016/" + str(m) + "/" + str(d) + "/DailyHistory.html"
      page = urllib2.urlopen(url)
 
      # Get temperature from page
      soup = BeautifulSoup(page)
      # dayTemp = soup.body.nobr.b.string
      dayTemp = soup.findAll(attrs={"class":"wx-value"})[0].contents[0].string
 
      # Format month for timestamp
      if len(str(m)) < 2:
        mStamp = '0' + str(m)
      else:
        mStamp = str(m)
 
      # Format day for timestamp
      if len(str(d)) < 2:
        dStamp = '0' + str(d)
      else:
        dStamp = str(d)
 
      # Build timestamp
      timestamp = '2016' + mStamp + dStamp
 
      # Write timestamp and temperature to file
      print timestamp + ',' + dayTemp + '
'

终端输入并运行文件

python get-weather-data.py

2.2.3 用代码来格式化

1. CSV转为XML

import csv

reader = csv.reader(open('wunder-data.txt', 'r'), delimiter=",")

print '<weather_data>'

for row in reader:
    print '<observation>'
    print '<date>' + row[0] + '</date>'
    print '<temperature>' + row[1] + '</temperature>'
    print '</observation>'

print '</weather_data>'

终端输入并运行文件

python csv2xml.py >wunder-data1.xml

f = open('wunder-data.xml', 'w')

f.write('<weather_data>')

for row in reader:
    f.write( '<observation>')
    f.write( '<date>' + row[0] + '</date>')
    f.write( '<temperature>' + row[1] + '</temperature>')
    f.write( '</observation>')

f.write( '</weather_data>')

f.close()

2. XML转为CSV

from BeautifulSoup import BeautifulStoneSoup

f = open('wunder-data.xml', 'r')
xml = f.read()
soup = BeautifulStoneSoup(xml)
observations = soup.findAll('observation')
for o in observations:
    print o.date.string + "," + o.temperature.string

终端输入并运行文件

python xml2csv.py >wunder-data1.txt

3. CSV转为JSON

import csv

reader = csv.reader(open('wunder-data.txt', 'r'), delimiter=",")

print '{ "observations": ['
rows_so_far = 0
for row in reader:
    
    rows_so_far += 1
    
    print '{' 
    print '"date": ' + '"' + row[0] + '", '
    print '"temperature": ' + row[1] 
    
    if rows_so_far < 365:
        print " },"
    else:
        print " }"
    
print "] }"

终端输入并运行文件

python csv2json.py >wunder-data1.json

4.在循环中加入新的逻辑

import csv

reader = csv.reader(open('wunder-data.txt', 'r'), delimiter=",")

for row in reader:
    if int(row[1]) <= 32:
        is_freezing = '1'
    else:
        is_freezing = '0'
    
    print row[0] + "," + row[1] + "," + is_freezing

终端输入并运行文件

python freezingInfo.py >wunder-data-fz.txt