本章内容
1、rawlog处理
2、域名item接口刷新
3、备案结果查询
4、多级域名中取主域
5、发送邮件
6、通过api获取cdn edge ip
7、多线程下载
1、rawlog处理
脚本里面涉及的内容
1、使用多cpu处理
2、UTC与GMT-8时间的转换
3、一个目录下对子目录的文件遍历
下面有多个panther-*目录
4、gzip文件的读取处理
Parsing Per-Hit (PerHit) Log using Python3 (incl. Multi-Thread version)
#!/usr/bin/env python3 # coding: utf-8 import os import sys import traceback import re import gzip from datetime import datetime from dateutil import tz base_path = "/home/xuanjia/static.trthi.com" file_name_prefix = "F114BC2216604A2C93AF5F6821168CA5_" file_name_sufix = "_pca_cn_cas_001.log" def conv_date(input_date, input_hour): local_date = datetime.strptime(input_date + " " + input_hour, "%Y%m%d %H").replace(tzinfo=tz.gettz('UTC')).astimezone(tz.gettz('Asia/Shanghai')) re_date = [] re_date.append(local_date.strftime("%Y%m%d")) re_date.append(local_date.strftime("%H")) return re_date def main(): global base_path global file_name_prefix global file_name_sufix array_subdirs = [] array_hours = [] array_subdirs = os.listdir(base_path) for i in range(0, 24, 1): array_hours.append('{0:02}'.format(i)) for hour in array_hours: for subdir in array_subdirs: f_name = conv_date(subdir, hour) print(f_name) with open(base_path + "/" + file_name_prefix + f_name[0] + "_" + f_name[1] + file_name_sufix, 'w') as output_f: # output_lines = [] for root, dirs, files in os.walk(base_path + "/" + subdir): for file_name in files: if re.search(".*_upstream_.*", file_name) is None and re.search(".*_" + subdir + "_" + hour + "_.*", file_name) is not None: try: with gzip.open(os.path.join(root + "/" + file_name), 'rt', encoding='utf-8') as input_f: for input_line in input_f: array_line = input_line.split(' ') dict_line = {} dict_line["Event-Type"] = array_line[0] dict_line["Site-ID"] = array_line[1] dict_line["Date"] = array_line[2] dict_line["Time"] = array_line[3] dict_line["C-IP"] = array_line[4] dict_line["CS-UserName"] = array_line[5] dict_line["S-SiteName"] = array_line[6] dict_line["S-ComputerName"] = array_line[7] dict_line["S-IP"] = array_line[8] dict_line["S-Port"] = array_line[9] dict_line["CS-Method"] = array_line[10] dict_line["CS-URI"] = array_line[11] dict_line["CS-URI-Query"] = array_line[12] dict_line["SC-Status"] = array_line[13] dict_line["SC-Win32-Status"] = array_line[14] dict_line["SC-Bytes"] = array_line[15] dict_line["CS-Bytes"] = array_line[16] dict_line["Time-Taken"] = array_line[17] dict_line["CS-Version"] = array_line[18] dict_line["CS-Host"] = array_line[19] dict_line["CS-UserAgent"] = array_line[20] dict_line["CS-Cookie"] = array_line[21] dict_line["CS-Referer"] = array_line[22] dict_line["SC-Sub-Status"] = array_line[23] dict_line["CS-Range"] = array_line[24] dict_line["SC-Initial"] = array_line[25] dict_line["SC-Complete"] = array_line[26] dict_line["SC-ContentType"] = array_line[27] dict_line["Protocol"] = array_line[28] dict_line["SC-Bytes-Body"] = array_line[29] dict_line["Bytes-Origin-Uncompressed"] = array_line[30] dict_line["C-RemotePort"] = array_line[31] # print(dict_line) array_output = [] array_output.append(dict_line["C-IP"]) array_output.append("-") array_output.append("-") array_output.append("[" + datetime.strptime(dict_line["Date"] + " " + dict_line["Time"], "%Y-%m-%d %H:%M:%S").replace(tzinfo=tz.gettz('UTC')).astimezone(tz.gettz('Asia/Shanghai')).strftime("%d/%b/%Y:%H:%M:%S +08:00") + "]") array_output.append(""" + dict_line["CS-Method"]) array_output.append("http://" + dict_line["CS-Host"] + dict_line["CS-URI"] + "?" + dict_line["CS-URI-Query"]) array_output.append(dict_line["CS-Version"] + """) array_output.append(dict_line["SC-Status"]) array_output.append(dict_line["SC-Bytes"]) array_output.append(""" + dict_line["CS-Referer"] + """) array_output.append(""" + dict_line["CS-UserAgent"] + """) array_output.append(""-"") array_output.append(dict_line["S-IP"]) # print(" ".join(array_output)) # output_lines.append(" ".join(array_output)) output_f.write(" ".join(array_output) + ' ') except Exception as e: traceback.print_exc(file=sys.stdout) print(root + "/" + file_name) continue exit() if __name__ == '__main__': main()
#!/usr/bin/env python3 # coding: utf-8 import os import sys import traceback import re import gzip import time import multiprocessing from datetime import datetime from dateutil import tz from multiprocessing import Pool base_path = "/home/xuanjia/static.trthi.com" file_name_prefix = "F114BC2216604A2C93AF5F6821168CA5_" file_name_sufix = "_pca_cn_cas_001.log" def conv_date(input_date, input_hour): local_date = datetime.strptime(input_date + " " + input_hour, "%Y%m%d %H").replace( tzinfo=tz.gettz('UTC')).astimezone(tz.gettz('Asia/Shanghai')) re_date = [] re_date.append(local_date.strftime("%Y%m%d")) re_date.append(local_date.strftime("%H")) return re_date def proc_log(proc_list): proc_files = proc_list[0] proc_filename = proc_list[1] with open(proc_filename, 'w') as output_f: for f in proc_files: try: with gzip.open(f, 'rt', encoding='utf-8') as input_f: for input_line in input_f: array_line = input_line.split(' ') dict_line = {} dict_line["Event-Type"] = array_line[0] dict_line["Site-ID"] = array_line[1] dict_line["Date"] = array_line[2] dict_line["Time"] = array_line[3] dict_line["C-IP"] = array_line[4] dict_line["CS-UserName"] = array_line[5] dict_line["S-SiteName"] = array_line[6] dict_line["S-ComputerName"] = array_line[7] dict_line["S-IP"] = array_line[8] dict_line["S-Port"] = array_line[9] dict_line["CS-Method"] = array_line[10] dict_line["CS-URI"] = array_line[11] dict_line["CS-URI-Query"] = array_line[12] dict_line["SC-Status"] = array_line[13] dict_line["SC-Win32-Status"] = array_line[14] dict_line["SC-Bytes"] = array_line[15] dict_line["CS-Bytes"] = array_line[16] dict_line["Time-Taken"] = array_line[17] dict_line["CS-Version"] = array_line[18] dict_line["CS-Host"] = array_line[19] dict_line["CS-UserAgent"] = array_line[20] dict_line["CS-Cookie"] = array_line[21] dict_line["CS-Referer"] = array_line[22] dict_line["SC-Sub-Status"] = array_line[23] dict_line["CS-Range"] = array_line[24] dict_line["SC-Initial"] = array_line[25] dict_line["SC-Complete"] = array_line[26] dict_line["SC-ContentType"] = array_line[27] dict_line["Protocol"] = array_line[28] dict_line["SC-Bytes-Body"] = array_line[29] dict_line["Bytes-Origin-Uncompressed"] = array_line[30] dict_line["C-RemotePort"] = array_line[31] array_output = [] array_output.append(dict_line["C-IP"]) array_output.append("-") array_output.append("-") array_output.append("[" + datetime.strptime(dict_line["Date"] + " " + dict_line["Time"], "%Y-%m-%d %H:%M:%S").replace( tzinfo=tz.gettz('UTC')).astimezone(tz.gettz('Asia/Shanghai')).strftime( "%d/%b/%Y:%H:%M:%S +08:00") + "]") array_output.append(""" + dict_line["CS-Method"]) array_output.append( "http://" + dict_line["CS-Host"] + dict_line["CS-URI"] + "?" + dict_line["CS-URI-Query"]) array_output.append(dict_line["CS-Version"] + """) array_output.append(dict_line["SC-Status"]) array_output.append(dict_line["SC-Bytes"]) array_output.append(""" + dict_line["CS-Referer"] + """) array_output.append(""" + dict_line["CS-UserAgent"] + """) array_output.append(""-"") array_output.append(dict_line["S-IP"]) output_f.write(" ".join(array_output) + ' ') except Exception as e: traceback.print_exc(file=sys.stdout) print(f) continue def main(): global base_path global file_name_prefix global file_name_sufix array_subdirs = [] array_hours = [] array_proc_files = [] array_params = [] time_s = time.time() c_count = multiprocessing.cpu_count() array_subdirs = os.listdir(base_path) #目录下的文件/目录，放到这个列表下面 【'20170912'，'20170913'】 for i in range(0, 24, 1): array_hours.append('{0:02}'.format(i)) #日期格式 01 02 03 for hour in array_hours: for subdir in array_subdirs: f_name = conv_date(subdir, hour) #utc 转换为加8的时间， 格式【'20170912'，'09'】 array_proc_files = [] for root, dirs, files in os.walk(base_path + "/" + subdir): for file_name in files: #file_name 会列出子目录下的所有文件 #收集需要统计的日志文件，之中不包括upstream的，和_文件夹下的 if re.search(".*_upstream_.*", file_name) is None and re.search(".*_" + subdir + "_" + hour + "_.*", file_name) is not None: array_proc_files.append(os.path.join(root + "/" + file_name)) if len(array_proc_files) > 0: #输出的目标文件 array_params.append((array_proc_files, base_path + "/" + file_name_prefix + f_name[0] + "_" + f_name[ 1] + file_name_sufix)) '''for param in array_params: print(param)''' with Pool(processes=c_count) as pool: pool.map(proc_log, array_params) time_e = time.time() time_delta = time_e - time_s print("Using " + str(time_delta)) exit() if __name__ == '__main__': main()
2、域名item接口刷新
脚本里面涉及的内容：
1、针对域名对uri做收集
2、收集1000uri后做处理
3、url的截取domain、uri
4、request post 多key相同的情况下请求
#!/usr/bin/env python3 #python version 3 import sys import requests from urllib.parse import urlparse username = 'Mr.python' #input your username password = '*******' #input your password mailto = 'Mr.python@txnetworks.cn' #input your email-address if len(sys.argv) != 2: print(' 33[1;32m You need input a filename,Format:*.py Filename !! 33[0m') sys.exit() filename = sys.argv[1] domains = {} with open(filename) as f: for line in f: if line == '': continue res = urlparse(line.strip()) parm = '' if res.query: parm = '?' + res.query pad, uri = res.netloc, res.path + parm #print('pad',pad,'uri',uri) if pad not in domains: domains[pad] = [] domains[pad].append(uri) else: domains[pad].append(uri) #print(domains) for domain in domains: print(' 33[1;32m Processing Domain: !! 33[0m',domain,'........') openapi = 'https://openapi.us.cdnetworks.com/purge/rest/doPurge' pad = domain flush_item = [] max_item = 1000 loop_count = 0 for uri in domains[domain]: if loop_count == max_item: payload = [('user',username),('pass',password),('pad',pad),('type','item'),('mailTo',mailto),('output','json')] payload = payload + flush_item payload = tuple(payload) #print(loop_count) #print(payload) res = requests.post(openapi,data=payload) print(res.text) loop_count = 1 flush_item = [] flush_item.append(('path',uri)) else: # print('loop_count',loop_count) flush_item.append(('path',uri)) #print(flush_item) loop_count +=1 if len(flush_item) != 0: payload = [('user',username),('pass',password),('pad',pad),('type','item'),('mailTo',mailto),('output','json')] payload = payload + flush_item payload = tuple(payload) #print(payload) res = requests.post(openapi,data=payload) print(res.text)
3、备案结果查询
脚本里面涉及的内容
1、client模块的使用
2、域名本案结果批量查询
#!/usr/bin/env python3 # coding: utf-8 import json from suds.client import Client import time def process(domains,begin,end): wsdl = "http://x.x.x.x:43392/?wsdl" param = json.dumps({"IcpRequest": {"domains": domains[begin:end]}}) client = Client(wsdl) client.set_options(timeout=3000) result_main = client.service.findDomainState_main(param) res = json.loads(result_main) res1 = res['IcpRespone']['domains'] for i in res1: print('Domain:%-30s NO:%-20s'%(i['domain'],i['phylicnum'])) with open('result.txt','a') as f: f.write(i['domain'] + ':' + i['phylicnum'] + ' ') def domains(filename): with open(filename) as f: domains = [] for line in f: domains.append(line.strip()) return domains def rangerequest(filename): domain = domains(filename) begin = 0 end = 10 n = 1 for i in range(0,len(domain),10): print(' 33[1;35m <======== begin:%s,end:%s , 第%s批 ========> 33[0m ' % (begin, end, n)) print(domain[begin:end]) print(' 33[1;31m Time:%s 33[0m'%time.strftime(" %X")) print('') try: process(domain,begin,end) except Exception as e: print(e) print('') print(' 33[1;32m done!!! 33[0m') begin += 10 end += 10 n += 1 if __name__ == '__main__': try: rangerequest('beian_domains.txt') #you need input you domain`s filename except Exception as e: print(e)
4、多级域名中取主域
脚本的设计内容
1、python中的正则
2、标准中定义的后缀
def get_sld(input_domain): output_domain = "" if re.search(r"^([0-9a-zA-Z-])+.([0-9a-zA-Z-])+$", input_domain): output_domain = input_domain elif re.search(r".(co|or|aaa|aarp|abb|abbott|abbvie|abogado|abudhabi|academy|accenture|accountant|accountants| aco|active|actor|adac|ads|adult|aeg|aero|afl|agakhan|agency|aig|airforce|airtel|akdn|allfinanz| ally|alsace|amica|amsterdam|analytics|android|anquan|apartments|app|aquarelle|aramco|archi|army| arpa|arte|asia|associates|attorney|auction|audi|audio|author|auto|autos|avianca|aws|axa|azure| baby|band|bank|bar|barcelona|barclaycard|barclays|barefoot|bargains|bauhaus|bayern|bbva|bcg|bcn| beats|beer|bentley|berlin|best|bet|bharti|bible|bid|bike|bing|bingo|bio|biz|black|blackfriday|bloomberg| blue|bms|bmw|bnl|bnpparibas|boats|boehringer|bom|bond|boo|book|boots|bosch|bostik|bot|boutique|bradesco| bridgestone|broadway|broker|brother|brussels|budapest|bugatti|build|builders|business|buy|buzz|bzh| cab|cafe|cal|call|camera|camp|cancerresearch|canon|capetown|capital|car|caravan|cards|care|career|careers| cars|cartier|casa|cash|casino|cat|catering|cba|cbn|ceb|center|ceo|cern|cfa|cfd|chanel|channel|chase|chat| cheap|chloe|christmas|chrome|church|cipriani|circle|cisco|citic|city|cityeats|claims|cleaning|click|clinic| clinique|clothing|cloud|club|clubmed|coach|codes|coffee|college|cologne|com|commbank|community|company|compare| computer|comsec|condos|construction|consulting|contact|contractors|cooking|cool|coop|corsica|country|coupon|coupons| courses|credit|creditcard|creditunion|cricket|crown|crs|cruises|csc|cuisinella|cymru|cyou| dabur|dad|dance|date|dating|datsun|day|dclk|dds|dealer|deals|degree|delivery|dell|deloitte|delta|democrat| dental|dentist|desi|design|dev|diamonds|diet|digital|direct|directory|discount|dnp|docs|dog|doha|domains|doosan| download|drive|dubai|durban|dvag| earth|eat|edeka|edu|education|email|emerck|energy|engineer|engineering|enterprises|equipment|erni| esq|estate|eurovision|eus|events|everbank|exchange|expert|exposed|express|extraspace| fage|fail|fairwinds|faith|family|fan|fans|farm|fashion|fast|feedback|ferrero|film|final|finance| financial|firestone|firmdale|fish|fishing|fit|fitness|flights|florist|flowers|flsmidth|fly|foo|football| ford|forex|forsale|forum|foundation|fox|fresenius|frl|frogans|frontier|ftr|fund|furniture|futbol|fyi| gal|gallery|gallo|gallup|game|garden|gbiz|gdn|gea|gent|genting|ggee|gift|gifts|gives|giving|glass|gle|global| globo|gmail|gmbh|gmo|gmx|gold|goldpoint|golf|goo|goog|gop|got|gov|grainger|graphics|gratis|green|gripe|group| gucci|guge|guide|guitars|guru| hamburg|hangout|haus|hdfcbank|health|healthcare|help|helsinki|here|hermes|hiphop|hitachi|hiv|hkt|hockey| holdings|holiday|homedepot|homes|honda|horse|host|hosting|hoteles|hotmail|house|how|hsbc|htc|hyundai| ibm|icbc|ice|icu|ifm|iinet|imamat|immo|immobilien|industries|infiniti|info|ing|ink|institute|insurance| insure|int|international|investments|ipiranga|irish|iselect|ismaili|ist|istanbul|itau|iwc| jaguar|java|jcb|jcp|jetzt|jewelry|jlc|jll|jmp|jnj|jobs|joburg|jot|jpmorgan|jprs|juegos| kaufen|kddi|kerryhotels|kerrylogistics|kerryproperties|kfh|kia|kim|kinder|kitchen|kiwi|koeln|komatsu|kpmg| kpn|krd|kred|kuokgroup|kyoto| lacaixa|lamborghini|lamer|lancaster|land|landrover|lanxess|lasalle|lat|latrobe|law|lawyer|lds|lease| leclerc|legal|lexus|lgbt|liaison|lidl|life|lifeinsurance|lifestyle|lighting|like|limited|limo|lincoln|linde| link|lipsy|live|living|lixil|loan|loans|locus|lol|london|lotte|lotto|love|ltd|ltda|lupin|luxe|luxury| madrid|maif|maison|makeup|man|management|mango|market|marketing|markets|marriott|mba|med|media|meet|melbourne| meme|memorial|men|menu|meo|miami|microsoft|mil|mini|mls|mma|mobi|mobily|moda|moe|moi|mom|monash|money|montblanc| mormon|mortgage|moscow|motorcycles|mov|movie|movistar|mtn|mtpc|mtr|museum|mutual|mutuelle| nadex|nagoya|name|natura|navy|nec|net|netbank|network|neustar|new|news|next|nextdirect|nexus|ngo|nhk|nico|nikon| ninja|nissan|nissay|nokia|northwesternmutual|norton|nowruz|nowtv|nra|nrw|ntt|nyc| obi|office|okinawa|olayan|olayangroup|omega|one|ong|onl|online|ooo|oracle|orange|org|organic|origins|osaka| otsuka|ovh| page|pamperedchef|panerai|paris|pars|partners|parts|party|passagens|pet|pharmacy|philips|photo|photography| photos|physio|piaget|pics|pictet|pictures|pid|pin|ping|pink|pizza|place|play|playstation|plumbing|plus|pohl| poker|porn|post|praxi|press|pro|prod|productions|prof|progressive|promo|properties|property|protection|pub|pwc| qpon|quebec|quest| racing|read|realtor|realty|recipes|red|redstone|redumbrella|rehab|reise|reisen|reit|ren|rent|rentals|repair| report|republican|rest|restaurant|review|reviews|rexroth|rich|ricoh|rio|rip|rocher|rocks|rodeo|room|rsvp|ruhr| run|rwe|ryukyu| saarland|safe|safety|sakura|sale|salon|sandvik|sandvikcoromant|sanofi|sap|sapo|sarl|sas|saxo|sbi|sbs|sca|scb| schaeffler|schmidt|scholarships|school|schule|schwarz|science|scor|scot|seat|security|seek|select|sener|services| seven|sew|sex|sexy|sfr|sharp|shaw|shell|shia|shiksha|shoes|shouji|show|shriram|singles|site|ski|skin|sky|skype| smile|sncf|soccer|social|softbank|software|sohu|solar|solutions|song|sony|soy|space|spiegel|spot|spreadbetting|srl| stada|star|starhub|statebank|statefarm|statoil|stc|stcgroup|stockholm|storage|store|stream|studio|study|style|sucks| supplies|supply|support|surf|surgery|suzuki|swatch|swiss|sydney|symantec|systems| tab|taipei|talk|taobao|tatamotors|tatar|tattoo|tax|taxi|tci|team|tech|technology|tel|telecity|telefonica| temasek|tennis|teva|thd|theater|theatre|tickets|tienda|tiffany|tips|tires|tirol|tmall|today|tokyo|tools|top| toray|toshiba|total|tours|town|toyota|toys|trade|trading|training|travel|travelers|travelersinsurance|trust|trv| tube|tui|tunes|tushu|tvs| ubs|unicom|university|uno|uol| vacations|vana|vegas|ventures|verisign|versicherung|vet|viajes|video|vig|viking|villas|vin|vip|virgin|vision| vista|vistaprint|viva|vlaanderen|vodka|volkswagen|vote|voting|voto|voyage|vuelos| wales|walter|wang|wanggou|warman|watch|watches|weather|weatherchannel|webcam|weber|website|wed|wedding|weir|whoswho |wien|wiki|williamhill|win|windows|wine|wme|wolterskluwer|work|works|world|wtc|wtf| xbox|xerox|xihuan|xin|орг|xperia|xxx|xyz| yachts|yahoo|yamaxun|yandex|yodobashi|yoga|yokohama|you|youtube|yun| zara|zero|zip|zone|zuerich){1}.(ac|ad|ae|af|ag|ai|al|am|an|ao|aq|ar|as|at|au|aw|ax|az| ba|bb|bd|be|bf|bg|bh|bi|bj|bm|bn|bo|bq|br|bs|bt|bv|bw|by|bz| ca|cc|cd|cf|cg|ch|ci|ck|cl|cm|cn|co|cr|cu|cv|cw|cx|cy|cz| de|dj|dk|dm|do|dz| ec|ee|eg|eh|er|es|et|eu| fi|fj|fk|fm|fo|fr| ga|gb|gd|ge|gf|gg|gh|gi|gl|gm|gn|gp|gq|gr|gs|gt|gu|gw|gy| hk|hm|hn|hr|ht|hu| id|ie|il|im|in|io|iq|ir|is|it| je|jm|jo|jp| ke|kg|kh|ki|km|kn|kp|kr|kw|ky|kz| la|lb|lc|li|lk|lr|ls|lt|lu|lv|ly| ma|mc|md|me|mg|mh|mk|ml|mm|mn|mo|mp|mq|mr|ms|mt|mu|mv|mw|mx|my|mz| na|nc|ne|nf|ng|ni|nl|no|np|nr|nu|nz| om| pa|pe|pf|pg|ph|pk|pl|pm|pn|pr|ps|pt|pw|py| qa| re|ro|rs|ru|rw| sa|sb|sc|sd|se|sg|sh|si|sj|sk|sl|sm|sn|so|sr|ss|st|su|sv|sx|sy|sz| tc|td|tf|tg|th|tj|tk|tl|tm|tn|to|tp|tr|tt|tv|tw|tz| ua|ug|uk|us|uy|uz| va|vc|ve|vg|vi|vn|vu| wf|ws| ye|yt| za|zm|zw){1}$", input_domain): match = re.search(r".{0,1}([0-9a-zA-Z-])+.(co|or| aaa|aarp|abb|abbott|abbvie|abogado|abudhabi|academy|accenture|accountant| accountants|aco|active|actor|adac|ads|adult|aeg|aero|afl|agakhan|agency| aig|airforce|airtel|akdn|allfinanz|ally|alsace|amica|amsterdam|analytics| android|anquan|apartments|app|aquarelle|aramco|archi|army|arpa|arte|asia|associates| attorney|auction|audi|audio|author|auto|autos|avianca|aws|axa|azure| baby|band|bank|bar|barcelona|barclaycard|barclays|barefoot|bargains|bauhaus|bayern| bbva|bcg|bcn|beats|beer|bentley|berlin|best|bet|bharti|bible|bid|bike|bing|bingo|bio|biz| black|blackfriday|bloomberg|blue|bms|bmw|bnl|bnpparibas|boats|boehringer|bom|bond|boo| book|boots|bosch|bostik|bot|boutique|bradesco|bridgestone|broadway|broker|brother|brussels| budapest|bugatti|build|builders|business|buy|buzz|bzh| cab|cafe|cal|call|camera|camp|cancerresearch|canon|capetown|capital|car|caravan|cards|care| career|careers|cars|cartier|casa|cash|casino|cat|catering|cba|cbn|ceb|center|ceo|cern|cfa|cfd| chanel|channel|chase|chat|cheap|chloe|christmas|chrome|church|cipriani|circle|cisco|citic|city| cityeats|claims|cleaning|click|clinic|clinique|clothing|cloud|club|clubmed|coach|codes|coffee| college|cologne|com|commbank|community|company|compare|computer|comsec|condos|construction| consulting|contact|contractors|cooking|cool|coop|corsica|country|coupon|coupons|courses|credit| creditcard|creditunion|cricket|crown|crs|cruises|csc|cuisinella|cymru|cyou| dabur|dad|dance|date|dating|datsun|day|dclk|dds|dealer|deals|degree|delivery|dell|deloitte| delta|democrat|dental|dentist|desi|design|dev|diamonds|diet|digital|direct|directory|discount| dnp|docs|dog|doha|domains|doosan|download|drive|dubai|durban|dvag| earth|eat|edeka|edu|education|email|emerck|energy|engineer|engineering|enterprises|equipment| erni|esq|estate|eurovision|eus|events|everbank|exchange|expert|exposed|express|extraspace| fage|fail|fairwinds|faith|family|fan|fans|farm|fashion|fast|feedback|ferrero|film|final|finance| financial|firestone|firmdale|fish|fishing|fit|fitness|flights|florist|flowers|flsmidth|fly|foo| football|ford|forex|forsale|forum|foundation|fox|fresenius|frl|frogans|frontier|ftr|fund|furniture| futbol|fyi| gal|gallery|gallo|gallup|game|garden|gbiz|gdn|gea|gent|genting|ggee|gift|gifts|gives|giving| glass|gle|global|globo|gmail|gmbh|gmo|gmx|gold|goldpoint|golf|goo|goog|gop|got|gov|grainger| graphics|gratis|green|gripe|group|gucci|guge|guide|guitars|guru| hamburg|hangout|haus|hdfcbank|health|healthcare|help|helsinki|here|hermes|hiphop|hitachi|hiv| hkt|hockey|holdings|holiday|homedepot|homes|honda|horse|host|hosting|hoteles|hotmail|house|how| hsbc|htc|hyundai| ibm|icbc|ice|icu|ifm|iinet|imamat|immo|immobilien|industries|infiniti|info|ing|ink|institute| insurance|insure|int|international|investments|ipiranga|irish|iselect|ismaili|ist|istanbul| itau|iwc| jaguar|java|jcb|jcp|jetzt|jewelry|jlc|jll|jmp|jnj|jobs|joburg|jot|jpmorgan|jprs|juegos| kaufen|kddi|kerryhotels|kerrylogistics|kerryproperties|kfh|kia|kim|kinder|kitchen|kiwi|koeln| komatsu|kpmg|kpn|krd|kred|kuokgroup|kyoto| lacaixa|lamborghini|lamer|lancaster|land|landrover|lanxess|lasalle|lat|latrobe|law|lawyer|lds| lease|leclerc|legal|lexus|lgbt|liaison|lidl|life|lifeinsurance|lifestyle|lighting|like|limited|limo| lincoln|linde|link|lipsy|live|living|lixil|loan|loans|locus|lol|london|lotte|lotto|love|ltd|ltda|lupin| luxe|luxury| madrid|maif|maison|makeup|man|management|mango|market|marketing|markets|marriott|mba|med|media| meet|melbourne|meme|memorial|men|menu|meo|miami|microsoft|mil|mini|mls|mma|mobi|mobily|moda|moe| moi|mom|monash|money|montblanc|mormon|mortgage|moscow|motorcycles|mov|movie|movistar|mtn|mtpc|mtr| museum|mutual|mutuelle| nadex|nagoya|name|natura|navy|nec|net|netbank|network|neustar|new|news|next|nextdirect|nexus|ngo| nhk|nico|nikon|ninja|nissan|nissay|nokia|northwesternmutual|norton|nowruz|nowtv|nra|nrw|ntt|nyc| obi|office|okinawa|olayan|olayangroup|omega|one|ong|onl|online|ooo|oracle|orange|org|organic| origins|osaka|otsuka|ovh| page|pamperedchef|panerai|paris|pars|partners|parts|party|passagens|pet|pharmacy|philips| photo|photography|photos|physio|piaget|pics|pictet|pictures|pid|pin|ping|pink|pizza|place|play| playstation|plumbing|plus|pohl|poker|porn|post|praxi|press|pro|prod|productions|prof|progressive|promo| properties|property|protection|pub|pwc| qpon|quebec|quest| racing|read|realtor|realty|recipes|red|redstone|redumbrella|rehab|reise|reisen|reit|ren|rent|rentals| repair|report|republican|rest|restaurant|review|reviews|rexroth|rich|ricoh|rio|rip|rocher|rocks|rodeo| room|rsvp|ruhr|run|rwe|ryukyu| saarland|safe|safety|sakura|sale|salon|sandvik|sandvikcoromant|sanofi|sap|sapo|sarl|sas|saxo|sbi|sbs| sca|scb|schaeffler|schmidt|scholarships|school|schule|schwarz|science|scor|scot|seat|security|seek| select|sener|services|seven|sew|sex|sexy|sfr|sharp|shaw|shell|shia|shiksha|shoes|shouji|show|shriram| singles|site|ski|skin|sky|skype|smile|sncf|soccer|social|softbank|software|sohu|solar|solutions|song| sony|soy|space|spiegel|spot|spreadbetting|srl|stada|star|starhub|statebank|statefarm|statoil|stc|stcgroup| stockholm|storage|store|stream|studio|study|style|sucks|supplies|supply|support|surf|surgery|suzuki|swatch| swiss|sydney|symantec|systems| tab|taipei|talk|taobao|tatamotors|tatar|tattoo|tax|taxi|tci|team|tech|technology|tel|telecity|telefonica| temasek|tennis|teva|thd|theater|theatre|tickets|tienda|tiffany|tips|tires|tirol|tmall|today|tokyo|tools|top| toray|toshiba|total|tours|town|toyota|toys|trade|trading|training|travel|travelers|travelersinsurance|trust| trv|tube|tui|tunes|tushu|tvs| ubs|unicom|university|uno|uol| vacations|vana|vegas|ventures|verisign|versicherung|vet|viajes|video|vig|viking|villas|vin|vip|virgin| vision|vista|vistaprint|viva|vlaanderen|vodka|volkswagen|vote|voting|voto|voyage|vuelos| wales|walter|wang|wanggou|warman|watch|watches|weather|weatherchannel|webcam|weber|website|wed|wedding| weir|whoswho|wien|wiki|williamhill|win|windows|wine|wme|wolterskluwer|work|works|world|wtc|wtf| xbox|xerox|xihuan|xin|орг|xperia|xxx|xyz| yachts|yahoo|yamaxun|yandex|yodobashi|yoga|yokohama|you|youtube|yun| zara|zero|zip|zone|zuerich){1}.(ac|ad|ae|af|ag|ai|al|am|an|ao|aq|ar|as|at|au|aw|ax|az| ba|bb|bd|be|bf|bg|bh|bi|bj|bm|bn|bo|bq|br|bs|bt|bv|bw|by|bz| ca|cc|cd|cf|cg|ch|ci|ck|cl|cm|cn|co|cr|cu|cv|cw|cx|cy|cz| de|dj|dk|dm|do|dz| ec|ee|eg|eh|er|es|et|eu| fi|fj|fk|fm|fo|fr| ga|gb|gd|ge|gf|gg|gh|gi|gl|gm|gn|gp|gq|gr|gs|gt|gu|gw|gy| hk|hm|hn|hr|ht|hu| id|ie|il|im|in|io|iq|ir|is|it| je|jm|jo|jp| ke|kg|kh|ki|km|kn|kp|kr|kw|ky|kz| la|lb|lc|li|lk|lr|ls|lt|lu|lv|ly| ma|mc|md|me|mg|mh|mk|ml|mm|mn|mo|mp|mq|mr|ms|mt|mu|mv|mw|mx|my|mz| na|nc|ne|nf|ng|ni|nl|no|np|nr|nu|nz| om| pa|pe|pf|pg|ph|pk|pl|pm|pn|pr|ps|pt|pw|py| qa| re|ro|rs|ru|rw| sa|sb|sc|sd|se|sg|sh|si|sj|sk|sl|sm|sn|so|sr|ss|st|su|sv|sx|sy|sz| tc|td|tf|tg|th|tj|tk|tl|tm|tn|to|tp|tr|tt|tv|tw|tz| ua|ug|uk|us|uy|uz| va|vc|ve|vg|vi|vn|vu| wf|ws| ye|yt| za|zm|zw){1}$", input_domain) if (str(match.group(0))[0:1] == "."): output_domain = str(match.group(0))[1:] else: output_domain = str(match.group(0)) elif re.search(r".(ac|ad|ae|af|ag|ai|al|am|an|ao|aq|ar|as|at|au|aw|ax|az| ba|bb|bd|be|bf|bg|bh|bi|bj|bm|bn|bo|bq|br|bs|bt|bv|bw|by|bz| ca|cc|cd|cf|cg|ch|ci|ck|cl|cm|cn|co|cr|cu|cv|cw|cx|cy|cz| de|dj|dk|dm|do|dz| ec|ee|eg|eh|er|es|et|eu| fi|fj|fk|fm|fo|fr| ga|gb|gd|ge|gf|gg|gh|gi|gl|gm|gn|gp|gq|gr|gs|gt|gu|gw|gy| hk|hm|hn|hr|ht|hu| id|ie|il|im|in|io|iq|ir|is|it| je|jm|jo|jp| ke|kg|kh|ki|km|kn|kp|kr|kw|ky|kz| la|lb|lc|li|lk|lr|ls|lt|lu|lv|ly| ma|mc|md|me|mg|mh|mk|ml|mm|mn|mo|mp|mq|mr|ms|mt|mu|mv|mw|mx|my|mz| na|nc|ne|nf|ng|ni|nl|no|np|nr|nu|nz| om| pa|pe|pf|pg|ph|pk|pl|pm|pn|pr|ps|pt|pw|py| qa| re|ro|rs|ru|rw| sa|sb|sc|sd|se|sg|sh|si|sj|sk|sl|sm|sn|so|sr|ss|st|su|sv|sx|sy|sz| tc|td|tf|tg|th|tj|tk|tl|tm|tn|to|tp|tr|tt|tv|tw|tz| ua|ug|uk|us|uy|uz| va|vc|ve|vg|vi|vn|vu| wf|ws| ye|yt| za|zm|zw){1}$", input_domain): match = re.search(r".{0,1}([0-9a-zA-Z-])+.(ac|ad|ae|af|ag|ai|al|am|an|ao|aq|ar|as|at|au|aw|ax|az| ba|bb|bd|be|bf|bg|bh|bi|bj|bm|bn|bo|bq|br|bs|bt|bv|bw|by|bz| ca|cc|cd|cf|cg|ch|ci|ck|cl|cm|cn|co|cr|cu|cv|cw|cx|cy|cz| de|dj|dk|dm|do|dz| ec|ee|eg|eh|er|es|et|eu| fi|fj|fk|fm|fo|fr| ga|gb|gd|ge|gf|gg|gh|gi|gl|gm|gn|gp|gq|gr|gs|gt|gu|gw|gy| hk|hm|hn|hr|ht|hu| id|ie|il|im|in|io|iq|ir|is|it| je|jm|jo|jp| ke|kg|kh|ki|km|kn|kp|kr|kw|ky|kz| la|lb|lc|li|lk|lr|ls|lt|lu|lv|ly| ma|mc|md|me|mg|mh|mk|ml|mm|mn|mo|mp|mq|mr|ms|mt|mu|mv|mw|mx|my|mz| na|nc|ne|nf|ng|ni|nl|no|np|nr|nu|nz| om| pa|pe|pf|pg|ph|pk|pl|pm|pn|pr|ps|pt|pw|py| qa| re|ro|rs|ru|rw| sa|sb|sc|sd|se|sg|sh|si|sj|sk|sl|sm|sn|so|sr|ss|st|su|sv|sx|sy|sz| tc|td|tf|tg|th|tj|tk|tl|tm|tn|to|tp|tr|tt|tv|tw|tz| ua|ug|uk|us|uy|uz| va|vc|ve|vg|vi|vn|vu| wf|ws| ye|yt| za|zm|zw){1}$", input_domain) if (str(match.group(0))[0:1] == "."): output_domain = str(match.group(0))[1:] else: output_domain = str(match.group(0)) elif re.search(r".([0-9a-zA-Z-])+.([0-9a-zA-Z-])+$", input_domain): match = re.search(r".([0-9a-zA-Z-])+.([0-9a-zA-Z-])+$", input_domain) output_domain = str(match.group(0))[1:] else: output_domain = input_domain return output_domain
5、发送邮件
脚本涉及的内容
1、126邮箱smtp服务端的设置
2、python发送邮件模块的使用
126服务端需要开启SMTP服务，这是你已经设置了客户端授权密码，这时候就需要用客户端授权密码去登录
# !/usr/bin/env python3 # -*- coding: UTF-8 -*- import smtplib from email.mime.text import MIMEText from email.header import Header sender = 'cangzhoufu@126.com' password = '******' subject = "CDN_Analysis - BEIAN Status Summary" fromaddr = "cangzhoufu@126.com" toaddrs = [ 'jin.wang@txnetworks.cn', 'zhongsheng@fastweb.com.cn'] #msg = ("From: %s Subject: %s To: %s " % (fromaddr, subject, ",".join(toaddrs))) #print(msg) subject = "python邮件测试" content = "这是我使用python smtplib及email模块发送的邮件" msg = MIMEText(content, 'plain', 'utf-8') msg['Subject'] = Header(subject, 'utf-8') msg['From'] = fromaddr msg['To'] = toaddrs #不能是个邮箱的列表得是单个邮箱,下面注释的是测试正确的，邮件中只会显示这个人的邮箱，列表中其他的邮箱被隐藏发送了 #subject = '放假通知' #msg =MIMEText(content,'plain','utf-8')#中文需参数‘utf-8'，单字节字符不需要 #msg['Subject'] = Header(subject, 'utf-8') #msg['From'] = 'cangzhoufu@126.com' #msg['To'] = "jin.wang@txnetworks.cn" try: smtpObj = smtplib.SMTP('smtp.126.com') smtpObj.set_debuglevel(1) smtpObj.login(sender, password) smtpObj.sendmail(sender, toaddrs, msg.as_string()) print("邮件发送成功") except smtplib.SMTPException as e: print("Error: 无法发送邮件",e) ============ 简洁版 ========== # -*- coding: UTF-8 -*- from email.mime.text import MIMEText from email.header import Header subject = '放假通知' msg = MIMEText('大家关好窗户','plain','utf-8')#中文需参数‘utf-8'，单字节字符不需要 msg['Subject'] = Header(subject, 'utf-8') msg['From'] = 'cangzhoufu@126.com' msg['To'] = "jin.wang@txnetworks.cn" from_addr = 'cangzhoufu@126.com' password = '*******' smtp_server = 'smtp.126.com' to_addr = 'jin.wang@txnetworks.cn' import smtplib server = smtplib.SMTP(smtp_server, 25) # SMTP协议默认端口是25 server.set_debuglevel(1) server.login(from_addr, password) server.sendmail(from_addr, [to_addr], msg.as_string()) server.quit()
收到的邮件内容如下：
6、通过api获取cdn edge ip
脚本涉及的内容：
1、获取token
2、再通过token获取ip list
3、requests
#!/usr/bin/env python3 import requests import json username = '1881062@163.com' password = '123!@#qwe' def get_token(): api_url = 'https://openapi.cdnetworks.com/api/rest/login' p_data = {'user':username,'pass':password,'output':'json'} r = requests.post(api_url,data=p_data) res = json.loads(r.text) if res['loginResponse']['resultCode'] == 0: return res else: return 'error' def pro_list(): result = get_token() for sess in result['loginResponse']['session']: print(sess) while True: svcGroupName = input('please input the svcGroupName:') for sess in result['loginResponse']['session']: if sess['svcGroupName'] == svcGroupName: #print('sessionToekn:',sess['sessionToken']) return sess['sessionToken'] else: continue print('your svcGroupName not correct!!!') def getCdnEdgeList(): sessionToken = pro_list() print(sessionToken) api_url = 'https://openapi.cdnetworks.com/api/rest/cdn/getCdnEdgeList' p_data = {'sessionToken': sessionToken, 'output': 'json'} r = requests.post(api_url,data=p_data) res = json.loads(r.text) if res['ipListResponse']['returnCode'] == 0: return res else: return 'error' def ip_list(): iplists = getCdnEdgeList()['ipListResponse']['item'] for ip in iplists: print(ip) if __name__ == '__main__': ip_list()
获取token后再获取apikey,两者结合再去查询
#!/usr/bin/env python3 import requests import json username = '1881061@163.com' password = '123!@#qwe' def get_token(): api_url = 'https://openapi.cdnetworks.com/api/rest/login' p_data = {'user':username,'pass':password,'output':'json'} r = requests.post(api_url,data=p_data) res = json.loads(r.text) if res['loginResponse']['resultCode'] == 0: return res else: return 'error' def pro_list(): result = get_token() for sess in result['loginResponse']['session']: print(sess) while True: svcGroupName = input('please input the svcGroupName:') for sess in result['loginResponse']['session']: if sess['svcGroupName'] == svcGroupName: #print('sessionToekn:',sess['sessionToken']) return sess['sessionToken'] else: continue print('your svcGroupName not correct!!!') def get_api_key_list(): post_data = {} sessionToken = pro_list() post_data['sessionToken'] = sessionToken print(sessionToken) api_key_list = 'https://openapi.cdnetworks.com/api/rest/getApiKeyList' post_data = {'sessionToken':sessionToken,'output':'json'} r = requests.post(api_key_list, data=post_data) res = json.loads(r.text) if res['apiKeyInfo']['returnCode'] == 0: for pad in res['apiKeyInfo']['apiKeyInfoItem']: print(pad) while True: padName = input('please input the PAD you want:') for sess in res['apiKeyInfo']['apiKeyInfoItem']: if sess['serviceName'] == padName.strip(): #print('sessionToekn:',sess['sessionToken']) choice_apikey = sess['apiKey'] post_data['apiKey'] = choice_apikey return post_data else: continue else: print('get data fail,Maybe there is something wrong with the sessionToken') def traffic_info(): post_data = get_api_key_list() traffic_info_api = 'https://openapi.cdnetworks.com/api/rest/traffic/edge' post_data['output'] = 'json' post_data['fromDate'] = 20171212 post_data['toDate'] = 20171213 post_data['timeInterval'] = 0 print(post_data) r = requests.post(traffic_info_api, data=post_data) res = json.loads(r.text) res = json.dumps(res, indent=4) print(res) def getCdnEdgeList(): sessionToken = pro_list() print(sessionToken) api_url = 'https://openapi.cdnetworks.com/api/rest/cdn/getCdnEdgeList' p_data = {'sessionToken': sessionToken, 'output': 'json'} r = requests.post(api_url,data=p_data) res = json.loads(r.text) if res['ipListResponse']['returnCode'] == 0: return res else: return 'error' def ip_list(): iplists = getCdnEdgeList()['ipListResponse']['item'] for ip in iplists: print(ip) if __name__ == '__main__': traffic_info()
8、多线程下载
脚本涉及的内容
一个大的文件，分range去分段请求，下载完毕后然后再组合到一起
#!/usr/bin/env python #coding:utf-8 from multiprocessing import Pool import sys import requests class downloader(object): # 构造函数 def __init__(self,url,num=4): # 要下载的数据连接 self.url = url # 要开的线程数 self.num = num # 存储文件的名字，从url最后面取 self.name = self.url.split('/')[-1] # head方法去请求url r = requests.head(self.url) # headers中取出数据的长度 self.total = int(r.headers['Content-Length']) print 'total is %s' % (self.total) def get_range(self): ranges=[] # 比如total是50,线程数是4个。offset就是12 offset = int(self.total/self.num) for i in range(self.num): if i==self.num-1: # 最后一个线程，不指定结束位置，取到最后 ranges.append((i*offset,'')) else: # 每个线程取得区间 ranges.append((i*offset,(i+1)*offset)) return ranges # range大概是[(0,12),(12,24),(25,36),(36,'')] def download(self,start,end): # 拼出Range参数 获取分片数据 headers={'Range':'Bytes=%s-%s' % (start,end),'Accept-Encoding':'*'} res = requests.get(self.url,headers=headers) print '%s:%s download success'%(start,end) #seek(m,n):从文件n位置开始，指针偏移m个字节，n:0(文件头),1(当前位置),2(文件未),seek(x),从x处开始 self.fd.seek(start) self.fd.write(res.content) def run(self): self.fd = open(self.name,'w') p = Pool(self.num) n = 1 for ran in self.get_range(): start,end = ran p.apply_async(self.download,args=(start,end,)) print 'Proces %s start:%s,end:%s'% (n,start,end) n += 1 p.close() p.join() print 'download %s load success'% (self.name) self.fd.close() if __name__=='__main__': #down = downloader('http://51reboot.com/src/blogimg/pc.jpg',5) if len(sys.argv) != 3: print "usage: python download2.py url num" sys.exit(1) down = downloader(sys.argv[1],int(sys.argv[2])) down.run()
9、服务器硬盘清理
脚本涉及的内容
Popen的使用，使用python在linux服务器上面执行一些shell命令
#!/usr/bin/python # Managed by Puppet - /var/cdn/ops/clearcache2.py # $Id: clearcache2.py 5526 2013-07-24 02:18:11Z taejoon.moon $ ##### See OPSUSSD-653 for the history # clearcache2.py -- New, more efficient clearcache script # # Improves performance over clearcache.sh by destroying and re-creating the # filesystem on each /cacheX partition instead of running a slow "rm -rf" # # Usage: sudo ./clearcache2.py ##### from subprocess import PIPE, Popen, STDOUT from os import getuid from time import sleep def stopServices(): print "===== Stopping HTTP and NMON =====" Popen(['/home/cdn/nmon/nmon', 'stop']).wait() Popen(['/home/cdn/http/http', 'stop']).wait() print "...done" def startServices(): print "===== Re-starting HTTP and NMON =====" Popen(['/home/cdn/http/http', 'restart']).wait() Popen(['/home/cdn/nmon/nmon', 'restart' ]).wait() print "...done" def prepareCache(): print "===== Preparing cache for first-use =====" print "Removing old index and flush informations" Popen(["/bin/rm -rf /var/cdn/http/cacheindex.* /var/cdn/http/flush* /var/cdn/http/last-flush-id /var/cdn/http/http-sites-verified.xml.*"], shell=True).wait() print "Cleaning JVM crash dumps" Popen(["/bin/rm -rf /tmp/hs_err_pid*"], shell=True).wait() print "Creating cacheindex.createit" Popen(['touch', '/var/cdn/http/cacheindex.createit']).wait() print "...done" def getPartitions(): """ Executes "df -T" and parses output, looking for /cache* partitions. Returns a dict where {mountpoint: device} A partition is considered a "Cache partition" if the following are met: 1.) The string "/cache" is found in the mountpoint name 2.) The filesystem type is ext3 """ partitions = {} p = Popen(['df', '-T'], stdout=PIPE, stderr=STDOUT) for line in p.stdout.readlines(): line = line.split() if "cache" in line[6] and line[1] == "ext3": partitions[line[6]] = line[0] return partitions def formatPartition(mountpoint, device): """ Unmounts, re-FS'es, and re-mounts a given partition with a given mountpoint. Filesystem label will be extracted from the mountpoint. This function assumes we're dealing with a /cacheX partition. """ print "===== Handling %s (%s) =====" % (mountpoint, device) print "Unmounting %s" % (mountpoint) Popen(['umount', '-f', mountpoint]).wait() print "...done" label = mountpoint.split("/")[1] print "Making ext3 fs on %s with label %s" % (device, label) Popen(['/sbin/mkfs', '-t', 'ext3', '-T', 'largefile4', '-L', label, device]).wait() print "...done" print "Re-mounting %s as %s" % (device, mountpoint) Popen(['mount', mountpoint]).wait() print "...done" print "Setting permissions on %s" % (mountpoint) Popen(['chown', 'http:cdn', mountpoint]).wait() print "...done" print "" if __name__ == "__main__": if getuid() is not 0: print "You must run this script as root" print "Usage: sudo clearcache2.py" else: print """ ***** WARNING ***** You are about to destroy ALL CACHED CONTENT on this node. If you do not want to flush this entire node, Ctrl-C now! ...Starting in 10 seconds... ******************* """ sleep(10) stopServices() for mountpoint, device in sorted(getPartitions().items()): #print "DEBUG %s as %s" % (mountpoint, device) formatPartition(mountpoint, device) prepareCache() startServices() print "" print "All tasks complete!"