zoukankan      html  css  js  c++  java
  • logstash grok 分割匹配日志

    使用logstash的时候,为了更细致的切割日志,会写一些正则表达式。 使用方法

            input {
            file {
                    type => "billin"
                    path => "/data/logs/product/result.log"
                }
            }
            filter  {
            grok    {
                type => "billin"
                pattern => "%{BILLINCENTER}"
                patterns_dir => "/data/logstash/patterns/my_patterns"
                } 
            }
            output  {
            redis   {
                host => "192.168.50.13"
                data_type =>"list"
                key => "logstash:redis"
            }
            }
    

    以下内容为正则表达式文件:cat my_patterns

            TAB 	
            META -+
            WZ ([^ ]*)
            IPPORT %{IP}:%{POSINT}|%{META}
            REQUEST (?:/[A-Za-z0-9$.+!*'(),~:#%_-]*)+?[A-Za-z0-9$.+!*'(),~#%&/=:;_-]*
            TY (?:(?<!\)(?:"(?:\.|[^\"]+)*"
            #EVERYURL ((w+://)?([^.]+)(.[^/:]+)(:d*)?([^#]*))|-
            #EVERYURL (((w+://)?([^.]+)(.[^/:]+)?([^#]*))+)|(w+)|-
            #EVERYURL ((w+://)?([^.]+)(.[^/:]+)?([^#]*))+)|-
            EVERYURL (http://+[wd:#@%/;$()~_?+-=\.&]+)|(-)
            #Logformat
            ########nginx access log example########
            #122.137.199.113"122.137.199.113"www.xxxx.com172.16.10.110172.16.12.114:8018/Jun/2013:15:51:03 +0800GET /g/getSaleCounts.do?rnd=1371541857448&showStatus=true&goodsIds=215abd2e8fa95bc8 HTTP/1.120078"http://www.xxxx.com/goods-215abd2e8fa95bc8.html""Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; NP06)""a8fdb711-a695-43bd-abdd-a224fb07350d"
            ###############################
            NGINXACCESSLOG %{IP:remote_ip}%{SPACE}%{QS:x_forward}%{SPACE}%{HOSTNAME:server_name}%{SPACE}%{IP:server_ip}%{SPACE}%{IPPORT:upstrem_ip}%{SPACE}%{HTTPDATE:timestamp}%{SPACE}%{WORD:verb}%{SPACE}%{WZ:request}%{SPACE}HTTP/%{NUMBER:httpversion}%{SPACE}%{NUMBER:response}%{SPACE}%{NUMBER:bytes}%{SPACE}%{QS:uri}%{SPACE}%{QS:agent}%{SPACE}%{QS:guid}
            #picture p0.xxxx.com access log . 2012.07.19 add
            PICLOG %{IP:remote_ip}%{SPACE}%{QS:x_forward}%{SPACE}%{HOSTNAME:server_name}%{SPACE}%{IP:server_ip}%{SPACE}%{HTTPDATE:timestamp}%{SPACE}%{WORD:verb}%{SPACE}%{WZ:request}%{SPACE}HTTP/%{NUMBER:httpversion}%{SPACE}%{NUMBER:response}%{SPACE}%{NUMBER:bytes}%{SPACE}%{QS:uri}%{SPACE}%{QS:agent}
            #iis log format 20120618 add
            ###########iis log example###############
            #2013-06-18 08:00:00 172.16.10.233 GET /js/functions.js - 80 - 117.136.34.2 Mozilla/5.0+(Linux;+U;+Android+4.1.2;+zh-CN;+LT22i+Build/6.2.A.0.400)+AppleWebKit/534.31+(KHTML,+like+Gecko)+UCBrowser/9.0.1.275+U3/0.8.0+Mobile+Safari/534.31 200 0 0 0
            ###################################
            IISLOG %{DATE_EU:log_date} %{TIME:log_time} %{IP:server_ip} %{WORD:verb} %{URIPATH:uri_stem} %{WZ:uri_query} %{POSINT:s_port} %{WZ:cs_username} %{IP:c_ip} %{WZ:agent} %{POSINT:request} %{POSINT:substatus} %{POSINT:win32_status} %{POSINT:time_taken}
            #2012/07/12 add
            ZW w+
            ###java date example
            #  2012-11-27 14:52:42
            ############
            JAVA_DATE %{DATE_EU} %{TIME}
            EARTHLOG [%{JAVA_DATE:log_date}] [%{WORD:level}] [%{WORD:action}] [{"desc":"%{ZW:desc}","dateTime":%{ZW:dateTime},"userId":"%{ZW:userId}","code":%{ZW:code}}]
            EAGLEUPDATE [%{JAVA_DATE:log_date}] [%{WORD:level}] [%{WORD:action}] [{"desc":%{QS:desc},"dateTime":%{ZW:dateTime},"userId":"%{ZW:userId}","code":%{ZW:code},"orderId":"%{ZW:orderId}"}]
            EAGLELOGIN [%{JAVA_DATE:log_date}] [%{WORD:level}] [%{WORD:action}] [{"desc":%{QS:desc},"dateTime":%{ZW:dateTime},"userId":"%{ZW:userId}","code":%{ZW:code}}]
            #2012/10/23 add
            LJF (-s+-)
            RESINLOG %{IP:remote_ip}%{SPACE}%{NUMBER}%{SPACE}%{LJF}%{SPACE}[%{HTTPDATE:timestamp}]%{SPACE}"%{WORD:verb}%{SPACE}%{WZ:request}%{SPACE}HTTP/%{NUMBER}"%{SPACE}%{NUMBER:response}%{SPACE}%{NUMBER:bytes}%{SPACE}%{QS:uri}%{SPACE}%{QS:agent}%{SPACE}%{QS:session}
            #RESINLOG %{IP:ip} %{NUMBER} - - [%{HTTPDATE:time}] "%{WORD:verb} %{WZ:request} HTTP/%{NUMBER}" %{NUMVER:response} %{NUMBER:bytes} %{QS:uri} %{QS:agent} %{QS:session}
            #2012/11/13 add
            DKH ({.*})
            STOREGREP ([/// - ] INFO  -)
            DHMH ([^;|=]*)
            CENTERLOG %{JAVA_DATE} %{STOREGREP} BId=%{NUMBER:bid};BR=%{DHMH:br};BP=%{DKH:bp}
            #2012/11/20 add
            JAVAGREP ([/// - ])
            ORDERCENTERERR %{JAVA_DATE} [ RMI TCP Connection(%{NUMER:threadid}) -%{IP:ip}] %{JAVAGREP} %{WORD:level}%{SPACE}%{WZ} - %{QS:message}
            ORDERCENTERRESULT %{JAVA_DATE} [ RMI TCP Connection(%{NUMER:threadid}) -%{IP:ip}] %{JAVAGREP} %{WORD:level}%{SPACE}%{WZ} - %{DKH:message}
            #2012/11/27 add
            #####log example#######
            #2013-06-18 15:28:12 INFO :{message:媒体传递的参数{"uid":["0"],"cid":["A100054947||0000"],"url":["http://www.xxxx.com/?from=lianmeng-weiyi"],"src":["weiyi"]}}
            #
            PARTNER %{JAVA_DATE:timestamp} %{WORD:level} :%{DKH:message}
            #2012/11/28 add
            PARTNERAPI %{JAVA_DATE:timestamp} %{WZ:level} :%{DKH:message}
            #2013/06/18 add
            #pattern all in the '[adskfjl }{]'
            FKH ([^;]*)
            #######aether.log#####
            #[2013-06-18 15:27:29] [INFO] [com.tuan.web.controller.IndexController] [{message:setHotStore#hot store size:5}]
            AETHERLOG [%{JAVA_DATE:timestamp}] [%{WZ:level}] [%{WZ:method}] %{FKH:message}
    
    
            USERNAME [a-zA-Z0-9._-]+
            USER %{USERNAME}
            INT (?:[+-]?(?:[0-9]+))
            BASE10NUM (?<![0-9.+-])(?>[+-]?(?:(?:[0-9]+(?:.[0-9]+)?)|(?:.[0-9]+)))
            NUMBER (?:%{BASE10NUM})
            BASE16NUM (?<![0-9A-Fa-f])(?:[+-]?(?:0x)?(?:[0-9A-Fa-f]+))
            BASE16FLOAT (?<![0-9A-Fa-f.])(?:[+-]?(?:0x)?(?:(?:[0-9A-Fa-f]+(?:.[0-9A-Fa-f]*)?)|(?:.[0-9A-Fa-f]+)))
    
            POSINT (?:[1-9][0-9]*)
            NONNEGINT (?:[0-9]+)
            WORD w+
            NOTSPACE S+
            SPACE s*
            DATA .*?
            GREEDYDATA .*
            #QUOTEDSTRING (?:(?<!\)(?:"(?:\.|[^\"])*"|(?:'(?:\.|[^\'])*')|(?:`(?:\.|[^\`])*`)))
            QUOTEDSTRING (?>(?<!\)(?>"(?>\.|[^\"]+)+"|""|(?>'(?>\.|[^\']+)+')|''|(?>`(?>\.|[^\`]+)+`)|``))
            UUID [A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12}
            # Networking
            MAC (?:%{CISCOMAC}|%{WINDOWSMAC}|%{COMMONMAC})
            CISCOMAC (?:(?:[A-Fa-f0-9]{4}.){2}[A-Fa-f0-9]{4})
            WINDOWSMAC (?:(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2})
            COMMONMAC (?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2})
            IPV6 ((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:)))(%.+)?
            IPV4 (?<![0-9])(?:(?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2}))(?![0-9])
            IP (?:%{IPV6}|%{IPV4})
            HOSTNAME (?:[0-9A-Za-z][0-9A-Za-z-]{0,62})(?:.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(.?|)
            HOST %{HOSTNAME}
            IPORHOST (?:%{HOSTNAME}|%{IP})
            HOSTPORT (?:%{IPORHOST=~/./}:%{POSINT})
            # paths
            PATH (?:%{UNIXPATH}|%{WINPATH})
            UNIXPATH (?>/(?>[w_%!$@:.,-]+|\.)*)+
            #UNIXPATH (?<![w/])(?:/[^/s?*]*)+
            TTY (?:/dev/(pts|tty([pq])?)(w+)?/?(?:[0-9]+))
            WINPATH (?>[A-Za-z]+:|\)(?:\[^\?*]*)+
            URIPROTO [A-Za-z]+(+[A-Za-z+]+)?
            URIHOST %{IPORHOST}(?::%{POSINT:port})?
            # uripath comes loosely from RFC1738, but mostly from what Firefox
            # doesn't turn into %XX
            URIPATH (?:/[A-Za-z0-9$.+!*'(){},~:;=@#%_-]*)+
            #URIPARAM ?(?:[A-Za-z0-9]+(?:=(?:[^&]*))?(?:&(?:[A-Za-z0-9]+(?:=(?:[^&]*))?)?)*)?
            URIPARAM ?[A-Za-z0-9$.+!*'|(){},~@#%&/=:;_?-[]]*
            URIPATHPARAM %{URIPATH}(?:%{URIPARAM})?
            URI %{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATHPARAM})?
    
            # Months: January, Feb, 3, 03, 12, December
            MONTH (?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)
            MONTHNUM (?:0?[1-9]|1[0-2])
            MONTHDAY (?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9])
    
            # Days: Monday, Tue, Thu, etc...
            DAY (?:Mon(?:day)?|Tue(?:sday)?|Wed(?:nesday)?|Thu(?:rsday)?|Fri(?:day)?|Sat(?:urday)?|Sun(?:day)?)
    
            # Years?
            YEAR (?>dd){1,2}
            # Time: HH:MM:SS
            #TIME d{2}:d{2}(?::d{2}(?:.d+)?)?
            # I'm still on the fence about using grok to perform the time match,
            # since it's probably slower.
            # TIME %{POSINT<24}:%{POSINT<60}(?::%{POSINT<60}(?:.%{POSINT})?)?
            HOUR (?:2[0123]|[01]?[0-9])
            MINUTE (?:[0-5][0-9])
            # '60' is a leap second in most time standards and thus is valid.
            SECOND (?:(?:[0-5][0-9]|60)(?:[:.,][0-9]+)?)
            TIME (?!<[0-9])%{HOUR}:%{MINUTE}(?::%{SECOND})(?![0-9])
            # datestamp is YYYY/MM/DD-HH:MM:SS.UUUU (or something like it)
            DATE_US %{MONTHNUM}[/-]%{MONTHDAY}[/-]%{YEAR}
            DATE_EU %{MONTHDAY}[./-]%{MONTHNUM}[./-]%{YEAR}
            ISO8601_TIMEZONE (?:Z|[+-]%{HOUR}(?::?%{MINUTE}))
            ISO8601_SECOND (?:%{SECOND}|60)
            TIMESTAMP_ISO8601 %{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}?
            DATE %{DATE_US}|%{DATE_EU}
            DATESTAMP %{DATE}[- ]%{TIME}
            TZ (?:[PMCE][SD]T|UTC)
            DATESTAMP_RFC822 %{DAY} %{MONTH} %{MONTHDAY} %{YEAR} %{TIME} %{TZ}
            DATESTAMP_OTHER %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{TZ} %{YEAR}
            # Syslog Dates: Month Day HH:MM:SS
            SYSLOGTIMESTAMP %{MONTH} +%{MONTHDAY} %{TIME}
            PROG (?:[w._/%-]+)
            SYSLOGPROG %{PROG:program}(?:[%{POSINT:pid}])?
            SYSLOGHOST %{IPORHOST}
            SYSLOGFACILITY <%{NONNEGINT:facility}.%{NONNEGINT:priority}>
            HTTPDATE %{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME} %{INT}
    
            # Shortcuts
            QS %{QUOTEDSTRING}
    
            # Log formats
            SYSLOGBASE %{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}:
            COMBINEDAPACHELOG %{IPORHOST:clientip} %{USER:ident} %{USER:auth} [%{HTTPDATE:timestamp}] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-) %{QS:referrer} %{QS:agent}
    
            # Log Levels
            LOGLEVEL ([A-a]lert|ALERT|[T|t]race|TRACE|[D|d]ebug|DEBUG|[N|n]otice|NOTICE|[I|i]nfo|INFO|[W|w]arn?(?:ing)?|WARN?(?:ING)?|[E|e]rr?(?:or)?|ERR?(?:OR)?|[C|c]rit?(?:ical)?|CRIT?(?:ICAL)?|[F|f]atal|FATAL|[S|s]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?)
    
  • 相关阅读:
    高格-远程支持中的奇怪问题【15】
    关于er图的几个工具
    如何解决win10明明是管理员还要权限的问题
    判断日期天数
    谈一谈在公司两次压测我总结的思路
    vue学习之-----v-model数据双向绑定,自定义组件父子传参
    Js各种小技巧总结
    openlayers学习之-----核心类
    openlayers学习之-----把坐标点改为WKT格式的数据
    新书介绍 -- 《Redis核心原理与实践》
  • 原文地址:https://www.cnblogs.com/shantu/p/4598875.html
Copyright © 2011-2022 走看看