python简单日志处理

简单日志处理

import datetime
import re
logfile='''58.61.164.141 - - [22/Feb/2010:09:51:46 +0800] "GET /reference-and-source/weblog-format/ HTTP/1.1" 200 6326 "-" "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)"'''

def resolve_log():
    _pattern = '''(?P<IP>[d.]{7,}) - - [(?P<datetime>[^[]]+)] "(?P<request>[^"]+)" (?P<status>d+) (?P<size>d+) "-" "(?P<useraAgent>[^"]+)"'''
    _regex = re.compile(_pattern, re.S)

    def _getHuman(size):
        units = ["","K","M","G","T","p"]
        deps = 0
        size = int(size)
        while(size >1000):
            deps += 1
            size = size // 1000
        return str(size)+units[deps]

    ops = {
        "datetime": lambda time:datetime.datetime.strptime(time,"%d/%b/%Y:%H:%M:%S %z"),
        "size": _getHuman,
        "status": int,
        "request": lambda request:dict(zip(("method","url","protocal"),request.split()))
    }

    def _extract(logfile):
        matcher = _regex.match(logfile)
        if matcher:
            return {k: ops.get(k, lambda x: x)(v) for k, v in matcher.groupdict().items()}
        else:
            return None
    return _extract

"""
    测试
"""
res = resolve_log()(logfile)
print(res)

"""
    可以打开一个文件测试代码
    open("www.log",mode="r",encoding="utf8")
"""

loged = []
with open("www.log",mode="rt",encoding="utf8") as f:
    for line in f:
        res = resolve_log()(line)
        loged.append(res)

处理结果：

{'size': '6K', 'datetime': datetime.datetime(2010, 2, 22, 9, 51, 46, tzinfo=datetime.timezone(datetime.timedelta(0, 28800))), 'IP': '58.61.164.141', 'status': 200, 'request': {'url': '/reference-and-source/weblog-format/', 'protocal': 'HTTP/1.1', 'method': 'GET'}, 'useraAgent': 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)'}