Python文件传参数和日志模块使用

　　说明：下面的脚本，涵盖利用文件传参数、logging日志模块，多进程

import requests
import re
import logging
from logging.handlers import RotatingFileHandler
import datetime
import time
import configparser
import os
import sys
import getpass
from multiprocessing import Pool
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)

def get_parameter():
    confFile='multiSiteAccess.cnf'
    if os.path.exists(confFile) is False:
        print('配置文件：{}不存在！'.format(confFile))
        opt=getpass.getpass('输入回车键退出程序！')
        sys.exit(1)
    else:
        cf=configparser.ConfigParser(allow_no_value=True)
        cf.read(confFile,encoding='UTF-8')
        try:
            username=cf.get('Auth','username')
            password=cf.get('Auth','password')
            urlFile=cf.get('FilePath','urlFile')
            proxyFile=cf.get('FilePath','proxyFile')
            ProccessNum=int(cf.get('Proccess','ProccessNum'))
            logFile=cf.get('logRecord','logFile')
            logPrint=cf.get('logRecord','logPrint')
            logRotateType=cf.get('logRecord','logRotateType')
            return username,password,urlFile,proxyFile,ProccessNum,logFile,logPrint,logRotateType
        except Exception as e:
            print(e)
            opt = getpass.getpass('输入回车键退出程序！')
            sys.exit(2)

def log_record_conf(logFile,logPrint,logRotateType):
    global logger
    logger=logging.getLogger()
    logger.setLevel(logging.DEBUG)
    console_handler=logging.StreamHandler()
    maxBytes=100*1024*1024
    interval=8
    if logRotateType.lower()=='size':
        file_handler=logging.handlers.RotatingFileHandler(filename=logFile,encoding='UTF-8',
                                                          maxBytes=maxBytes,backupCount=50,delay=False)
    elif logRotateType.lower()=='time':
        file_handler=logging.handlers.TimedRotatingFileHandler(filename=logFile,encoding='UTF-8',when='H',
                                                               interval=interval,delay=False,utc=False,atTime=datetime.time)
    else:
        file_handler=logging.FileHandler(filename=logFile,encoding='UTF-8',mode='a',delay=False)
    formatter=logging.Formatter('%(asctime)s - %(levelname)-7s - %(message)s')
    console_handler.setFormatter(formatter)
    file_handler.setFormatter(formatter)
    logging.addHandler(file_handler)
    if logPrint.lower()=='on':
        logger.addHandler(console_handler)
    return logger

def get_urls(urlFile):
    with open(urlFile,'r',encoding='UTF-8') as rf:
        allUrl=rf.readlines()
    return allUrl

def get_proxy_host(proxyFile):
    with open(proxyFile,'r',encoding='UTF-8') as rf:
        allProxy=rf.readlines()
    return allProxy

def url_sample_disposal(urllink):
    urllink=urllink.strip('"')
    urllink=urllink.strip("'")
    pattern=r'^https?://.*'
    match=re.findall(pattern,urllink)
    if len(match) ==0:        # URL样本不是以http/https开头的样本
        urllink1='http://{}'.format(urllink)
        urllink2='https://{}'.format(urllink)
        match.append(urllink1)
        match.append(urllink2)
    return match

def set_proxy(urlProxy,username,password):
    if not urlProxy:
        proxies={}
    else:
        http_proxy='http://{}:{}@{}'.format(username,password,urlProxy)
        https_proxy=http_proxy
        proxies={
            'http':http_proxy,
            'https':https_proxy
        }
    return proxies

def http_request(urllink,proxy='',proxyHost=''):
    headers={
        'User-Agent':'curl/3.03',
        'Connection':'close'    # keep-alive
    }
    requests.adapters.DEFAULT_RETRIES=5
    try:
        r=requests.get(urllink,headers=headers,proxies=proxy,timeout=30,verify=False)
        r.close()
        urlresult='{}	{}	{}'.format(proxyHost,r.url,str(r.status_code))
    except Exception as e:
        urlresult='{}	{}	{}'.format(proxyHost,urllink,str(e))
    finally:
        return urlresult

def writer_log(urlresult):
    logger.info(urlresult)

if __name__=='__main__':
    start_time=datetime.datetime.now()
    username,password,urlFile,proxyFile,ProccessNum,logFile,logPrint,logRotateType=get_parameter()
    logger=log_record_conf(logFile,logPrint,logRotateType)
    logger.info('{}URL测试开始{}'.format('='*15,'='*15))
    allUrl=get_urls(urlFile)
    allProxy=get_proxy_host(proxyFile)
    logger.debug('测试总URL数：{}'.format(len(allUrl)))
    logger.debug('测试进程数：{}'.format(ProccessNum))
    p=Pool(ProccessNum)
    for i in allUrl:
        urlLink=i.split()[-1]
        match=url_sample_disposal(urlLink)
        for urllink in match :
            urllink=urllink.strip()
            for proxyHost in allProxy:
                proxyHost=proxyHost.strip()
                proxies=set_proxy(proxyHost,username,password)
                result=p.apply_async(http_request,args=(urllink,proxies,proxyHost),callback=writer_log)
    p.close()
    count=0
    while True:
        try:
            time.sleep(60)
            if result.ready():
                count+=1
                time.sleep(30)
                logger.debug('进程池调度结束')
                result.get(timeout=30)
            if count>=3:
                break
        except Exception as e:
            logger.warning('进程异常：{}'.format(e))
    p.terminate()
    p.join()
    end_time=datetime.datetime.now()
    logger.info('开始时间：{}'.format(start_time))
    logger.info('结束时间：{}'.format(end_time))
    logger.info('总耗时：{}'.format(end_time-start_time))
    logger.debug('日志文件：{}'.format(logFile))
    logger.info('{}URL测试结束{}'.format('=' * 15, '=' * 15))