实时爬取上海快3的结果

对于静态页面可以用request获取当前页面上html的源代码，这里不多说了，代码中也已经注释。方法也比较简单，本人觉得现在的网站多数是采用的动态页面，所以如何爬取js动态加载后的html内容还是比较重要的。以上海快3开奖结果为例，实时的爬取快3的开奖结果，代码如下：

# -*- coding:utf-8 -*-
from bs4 import BeautifulSoup
import datetime,time
from selenium.webdriver import Firefox
from selenium.webdriver.firefox.options import Options
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
import logging
import traceback
from retrying import retry

now_time = datetime.datetime.now()



url='http://fucai.eastday.com/LotteryNew/K3Result.aspx'
# user_agent={
#     "User-Agent":"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:48.0) Gecko/20100101 Firefox/48.0"}
# headers = {"User-Agent": user_agent }

#用无图形化Firefox来获取当前页面的html，所以动态静态都一手掌控，稳定性很好，而且不容易被反爬虫拦截
@retry
def get_html(url):
    options = Options()
    options.add_argument('-headless')  # 无头参数
    dcap = dict(DesiredCapabilities.FIREFOX)
    dcap['FIREFOX.page.settings.userAgent'] = (
    'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36')
    browser = Firefox(firefox_options=options,desired_capabilities=dcap)
    time.sleep(2)
    browser.get(url)
    htmlPage=browser.page_source
    browser.close()
    return htmlPage

# def get_html(url):
#     htmlPage=requests.get(url).text
#     return htmlPage


#下一步就是根据获取到的html来做分析，取出我们需要的开奖结果的数值
def analyze(htmlPage):
    soup = BeautifulSoup(htmlPage, "lxml")
    # print soup.prettify()
    # tagTable = soup.find('table', classname='san-sheet-alternating-2')
    # result=soup.find_all(attrs={"class": "td_title02"})
    # result = soup.find(text=re.compile('ds*,s*ds*,s*d'))
    results = soup.find_all('span')[:3]
    list=[]
    for i in results:
        list.append(i.string)
    result=list[0]+','+list[1]+','+list[2]
    now_time = datetime.datetime.now()
    print now_time,u'开奖结果为:',result


#通过while循环来做到实时获取
while 1:
    htmlPage = get_html(url)
    analyze(htmlPage)
    time.sleep(60)