python超链接抓取工具

python实现自动抓取某站点内所有超链接

（仅供学习使用）

代码部分

#!/usr/bin/python

import requests
import time
import re
import sys, getopt  #命令行选项
from bs4 import BeautifulSoup
localtime=time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) #时间
z=[] #存取网站
x=[] #优化网站，去除冗杂部分

def main(argv):
   url = '' #输入的网址
   file_path = '' #保存路径
   try:
      opts, args = getopt.getopt(argv,"hu:f:",["url=","file="])
   except getopt.GetoptError:
      print ('allsite.py -u <url> -f <file>')
      sys.exit(2)
   for opt, arg in opts:
      if opt == '-h': #帮助
         print ('allsite.py -u <url> -f <file>')
         sys.exit()
      elif opt in ("-u", "--url"): #输入网址
         url = arg
         re1 = requests.get(url) #get网站内容
         re1.encoding = "utf-8"
         html = re1.text
         bt = BeautifulSoup(html, 'html.parser', )
         hh = bt.find_all('a') #查找<a>元素
         for site in hh:
                 z.append(site.get('href')) #进一步过滤得到超链接
         for i in z:
                 if (re.match('//www', str(i)) or re.match('www', str(i))): 
                         xx = str(i).replace('//www', 'www', 1)
                         x.append(xx)
                 elif (re.match('http', str(i))): #过滤
                         x.append(str(i))
                 elif (re.match('/', str(i))): #过滤
                         xx = str(i).replace("/", "", 1)
                         if (re.match('/', xx)):
                                 xxx = str(xx).replace("/", "", 1)
                                 x.append(xxx)
                         else:
                                 x.append(url + xx)
                 else: #过滤
                         if (re.search('javascript', str(i)) == None):
                                 x.append(url + str(i))
         print(localtime + "  总共:" + str(len(x)) + "个网址") #输出超链接
         for i in x:
                 print(i)
      elif opt in ("-f", "--file"): #输入保存路径
         file_path = arg
         for i in x: #保存文件
                 with open(file_path, 'a') as file_object:
                         file_object.write(i)
                         file_object.write('
')

if __name__ == "__main__":
   main(sys.argv[1:])