Python抓取双色球数据

　　数据来源网站http://baidu.lecai.com/lottery/draw/list/50?d=2013-01-01

　　HTML解析器http://pythonhosted.org/pyquery/ (可以像JQuery那样使用)

　　源码:

import MySQLdb as mysql
from pyquery import PyQuery as pq

create_table_sql = '''
create table union_lotto(
    issue int  primary key,
    lottery_date   date,
    lottery_number varchar(30)
)'''

sql = "insert into union_lotto values(%(issue)s, %(date)s, %(number)s)"
conn = mysql.connect(host='localhost', db='caipiao', user='root', passwd='')
cur = conn.cursor()


def inserts(rows):
    cur.executemany(sql, rows)
    conn.commit()


def close():
    conn.close()


def handler_row(row):
    children = row.getchildren()
    date =  children[0].text_content()
    issue = children[1].getchildren()[0].text_content()
    spans = children[2].getchildren()[0].getchildren()
    numbers = []
    for span in spans:
        numbers.append(span.text_content())
    lottery_number = '-'.join(numbers)
    return {'issue': int(issue.strip()), 'date': date, 'number': lottery_number}


def grab_data(url):
    d = pq(url=url)
    rows = d("#draw_list > tbody > tr")
    result = []
    for row in rows:
        result.append(handler_row(row))
    return result


def main():
    years = [(2003 + i) for i in range(0, 11)]
    url = 'http://baidu.lecai.com/lottery/draw/list/50?d=%d-01-01'
    print '.......star.........'
    for year in years:
        result = grab_data(url % year)
        inserts(result)
    close()
    print '.......end..........'


if __name__ == '__main__':
    main()