python selenium爬虫工具

今天seo的同事需要一个简单的爬虫工具， 根据一个url地址，抓取改页面的a连接，然后进入a连接里面的页面再次抓取a连接

1.需要一个全局的set([])集合来保存抓取的url地址

2.由于现在单页面也来越多，所以我们借用selenium来抓取页面内容， 由于页面内容比较多， 我们程序需要将滚动条滚到最下面，如：driver.execute_script("return document.body.scrollHeight;")

3.需要查找页面的超链接 driver.find_elements_by_xpath("//a[@href]")

4.为了便于查看数据记录，每抓取一个地址就记录到日志中去（曾经尝试过爬网完毕后再记录，但是爬网时间太长，一旦出现异常就一条记录都没有了）

整个代码如下：

from selenium import webdriver
from selenium.webdriver.firefox.options import Options
from selenium.common.exceptions import TimeoutException
import time
import datetime
from urllib import parse
import os

urls = set([])
def getUrl(url,host):
    driver = webdriver.Ie()
    try:
       #driver = webdriver.Firefox()
        driver.set_page_load_timeout(10)
        driver.get(url)
        #time.sleep(2)
        
        all_window_height = []
        all_window_height.append(driver.execute_script("return document.body.scrollHeight;"))
        while True:
            driver.execute_script("scroll(0,100000)")
            time.sleep(1)
            check_height = driver.execute_script("return document.body.scrollHeight;")
            if check_height == all_window_height[-1]:
                print("我已下拉完毕")
                break
            else:
                all_window_height.append(check_height) 
                print("我正在下拉")
        
        #for link in driver.find_elements_by_xpath("//*[@href]"):
        #for link in driver.find_elements_by_tag_name("a"):
        for link in driver.find_elements_by_xpath("//a[@href]"):
            try:
                tempurl1=link.get_attribute('href')
                if tempurl1.startswith("http"):
                    if tempurl1 not in urls:
                        urls.add(tempurl1)
                        log(host,url+','+tempurl1)
                        print(tempurl1)
            except:
                print(link)
    except Exception as e:
        print(e)
    finally:
        driver.quit()



def log(name,msg):
    filename='D://'+name+'.csv'
    if not os.path.exists(filename):
        with open(filename,'w') as f:
            print('create file:'+filename)
            f.write('parentUrl,currenturl'+'
')
        f.close()
    with open(filename,'a') as f:
        f.write(msg+'
')
    f.close()

url= input("Enter a url")
try:
    urls.clear()
    url= url.strip()
    if len(url)>0:
        host =parse.urlparse(url).netloc
        print(url+"下面的连接:")
        t1=datetime.datetime.now()
        getUrl(url,host)
        l=list(urls)
        for item in l:
            print(item+"下面的连接:")
            getUrl(item,host)
        t2=datetime.datetime.now()
        tt =(t2-t1).seconds
        minutes=tt//60
        seconds=tt%60
    print("total cost %d minutes %d seconds" % (minutes,seconds))

except Exception as e:
    print(e)

然后运行pyinstaller -F a.py 打包

关于selenium 的IE 可以参考https://blog.csdn.net/ma_jiang/article/details/96022775