常用的小脚本

1、检测表格中的网站是否能v6访问，是否有AAAA记录

#coding:utf8
import pycurl,xlrd,socket,xlwt
from io import BytesIO
from xlutils.copy import copy
import threadpool
import threading


look = threading.Lock()
#检测站点是否可以使用v6访问
def web_can_ipv6(url):
    buffer = BytesIO()
    c = pycurl.Curl()
    c.setopt(pycurl.OPT_CERTINFO, 1)
    c.setopt(pycurl.URL, "http://"+url)
    #设置只使用v6访问
    c.setopt(pycurl.IPRESOLVE,pycurl.IPRESOLVE_V6)
    c.setopt(c.WRITEDATA, buffer)
    try:
        c.perform()
        return c.getinfo(pycurl.HTTP_CODE)
    except:
        return None

#读取表格内容
def read_excel():
    web_list = []
    # 打开文件
    workbook = xlrd.open_workbook(r'webs.xls')
    # 获取所有sheet
    sheet1 = workbook.sheet_by_name('Sheet1')
    for row_n in range(1,sheet1.nrows):
        row = sheet1.row_values(row_n)
        web_list.append(row)
    return web_list

def write_excel(row,col,text):
    rb = xlrd.open_workbook(r'webs.xls','wb')  # 打开weng.xls文件
    wb = copy(rb)  # 利用xlutils.copy下的copy函数复制
    ws = wb.get_sheet(0)  # 获取表单0
    ws.write(row, col, text)
    wb.save(r'webs.xls')  # 保存文件

#返回站点的AAAA记录
def get_dns(url):
    try:
        results = socket.getaddrinfo(url, None)
        for item in results:
            if ':' in item[4][0]:
                return item[4][0]
    except:
        print('%s-error'%url)


def main(web):
    print('正在检测第%s个网站' % web[0])
    write_excel(web[0],3,get_dns(web[2]))
    if web_can_ipv6(web[2]):
        write_excel(web[0],4, web_can_ipv6(web[2]))





if __name__ == '__main__':
    for web in read_excel():
        main(web)

 2、检测表格中的网站的返回码，归属运营商，多线程

总共一万多条数据的表格，采用分段处理的方式，大大增加处理速度

##coding:utf8
import threadpool
import threading
import xlrd,xlwt,requests,socket
from xlutils.copy import copy
from IPy import IP
from contextlib import closing
from concurrent.futures import ThreadPoolExecutor,wait

lock = threading.Lock()
pool = threadpool.ThreadPool(20)

#获取网站返回码
def get_web_code(ws,web):
    try:
        url = str(web[2]).strip()
        url = 'http://'+url
        headers = {
                'User-Agent': 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0) ',
            }
        r = requests.head(url=url,timeout=1,headers=headers)

        print('第%s行-->%s' % (web[0], str(r.status_code)))

        if is_url_in_cernet(get_dns(url)):
            write_excel(ws,int(web[0]), 4, 'CERNET')
        elif is_url_in_cmcc(get_dns(url)):
            write_excel(ws,int(web[0]), 4, 'CMCC')
        elif is_url_in_cucc(get_dns(url)):
            write_excel(ws,int(web[0]), 4, 'CUCC')
        elif is_url_in_ctcc(get_dns(url)):
            write_excel(ws,int(web[0]), 4, 'CTCC')
        else:
            write_excel(ws,int(web[0]), 4, 'UNKnow')
        write_excel(ws,int(web[0]),3,r.status_code)
        return r.status_code
    except Exception as e:
        write_excel(ws,int(web[0]),3,'ERROR')
        print('第%s行-->%s'%(web[0],e))


#读取表格数据
def read_excel():
    count=0
    web_list = []
    # 打开文件
    workbook = xlrd.open_workbook(r'webs.xls')
    # 获取所有sheet
    sheet1 = workbook.sheet_by_name('Sheet1')
    for row_n in range(1,sheet1.nrows):
        row = sheet1.row_values(row_n)
        web_list.append(row)
    n=100
    new_web_list = [web_list[i:i+n] for i in range(0,len(web_list),n)]
    return new_web_list


# #表格写入数据
# def write_excel(row,col,text):
#     lock.acquire()
#     try:
#         rb = xlrd.open_workbook(r'webs.xls','wb')
#         wb = copy(rb)  # 利用xlutils.copy下的copy函数复制
#         ws = wb.get_sheet(0)  # 获取表单0
#         ws.write(row, col, text)
#         wb.save(r'webs.xls')  # 保存文件
#     except:
#         pass
#     finally:
#         lock.release()


#表格写入数据
def write_excel(ws,row,col,text):
    lock.acquire()
    try:
        ws.write(row, col, text)
    except:
        pass
    finally:
        lock.release()


#返回站点的A记录
def get_dns(url):
    lock.acquire()
    if 'http://' or 'https://' in  url.lower():
        url = url.strip()
        url = url.lower()
        url = url.replace('http://','')
        url = url.replace('https://', '')
    if ':' in url:
        url = url.split(':')[0]
    try:
        results = socket.getaddrinfo(url, None)
        return results[0][4][0]
    except:
        return url
    finally:
        lock.release()

def is_url_in_cernet(url):
    lock.acquire()
    try:
        with open('cernet_ip.txt','r') as rf:
            for line in rf.readlines():
                if ';' in line:
                    line = line.replace(';', '')
                if ':' in line:
                    line = line.split(':')[0]
                ip_line = IP(line)
                if url in ip_line:
                    return True
    except:
        pass
    finally:
        lock.release()


def is_url_in_cmcc(url):
    lock.acquire()
    try:
        with open('CMCC_ip.txt','r') as rf:
            for line in rf.readlines():
                if ';' in line:
                    line = line.replace(';','')
                if ':' in line:
                    line = line.split(':')[0]
                ip_line = IP(line)
                if url in ip_line:
                    return True
    except:
        pass
    finally:
        lock.release()

def is_url_in_cucc(url):
    lock.acquire()
    try:
        with open('CUCC_ip.txt','r') as rf:
            for line in rf.readlines():
                if ';' in line:
                    line = line.replace(';', '')
                if ':' in line:
                    line = line.split(':')[0]
                ip_line = IP(line)
                if url in ip_line:
                    return True
    except:
        pass
    finally:
        lock.release()

def is_url_in_ctcc(url):
    lock.acquire()
    try:
        with open('CTCC_ip.txt','r') as rf:
            for line in rf.readlines():
                if ';' in line:
                    line = line.replace(';', '')
                if ':' in line:
                    line = line.split(':')[0]
                ip_line = IP(line)
                if url in ip_line:
                    return True
    except:
        pass
    finally:
        lock.release()




if __name__ == '__main__':
    new_web_list = read_excel()
    for web_list in new_web_list:
        rb = xlrd.open_workbook(r'webs.xls','wb')
        wb = copy(rb)  # 利用xlutils.copy下的copy函数复制
        ws = wb.get_sheet(0)  # 获取表单0
        t_list = []
        for web in web_list:
            work = threading.Thread(target=get_web_code , args=(ws, web,))
            work.start()
            t_list.append(work)
        for t in t_list:
            t.join()
        wb.save(r'webs.xls')
    print('-----end-----')
    # fs = []
    # for web in web_list:
    #     f = pool.submit(get_web_code, web)
    #     fs.append(f)
    # # 相遇与join，阻塞主线程
    # wait(fs)

 3、将n个表格中的m个子表的数据全部汇总到一个总表中

测试是将30+个表个汇总，耗时不到2分钟

# -*- coding: utf-8 -*- 
import os,xlrd
from xlutils.copy import copy
import time,re

#获取需要的表格
def get_file():
    file_list=[]
    for root,dirs,files in os.walk(os.getcwd()):
        for file in files:
            if 'excel_run.' in file:
                pass
            elif file=='total.xls':
                pass
            elif file=='~$total.xls':
                pass
            else:
                if re.search(r'/*.xls/*',file):
                    file_list.append(file)
    return file_list


#读取表格数据
def read_excel(filename,sheetindex):
    web_list = []
    # 打开文件
    workbook = xlrd.open_workbook(r'%s'%filename)
    # 获取所有sheet
    sheet1 = workbook.sheet_by_index(sheetindex)
    for row_n in range(2,sheet1.nrows):
        row = sheet1.row_values(row_n)
        web_list.append(row)
    return web_list


#表格写入数据
def write_excel(sheetindex,file,count):
    rb = xlrd.open_workbook(r'total.xls','wb')
    wb = copy(rb)  # 利用xlutils.copy下的copy函数复制
    ws = wb.get_sheet(sheetindex)  # 获取表单0
    nrow = rb.sheet_by_index(sheetindex).nrows
    for row in read_excel(file,sheetindex):
        count+=1
        for index_col,col in enumerate(row):
            ws.write(nrow+count, index_col, col)
    wb.save(r'total.xls')  # 保存文件


if __name__ == '__main__':
    t1 = time.time()
    for file in get_file():
        for sheetindex in [1,2,4,5,6]:
            count=-1
            print('正在录入%s的第%s张表的数据' % (file, sheetindex))
            write_excel(sheetindex,file,count)

    t2=time.time()
    print('共耗时%s秒'%(t2-t1))
    print('---end---')