logstash 正则调试；
nginx 配置；
log_format main '$remote_addr [$time_local] "$request" ';
logstash:
"message" =>"%{IPORHOST:clientip} [%{HTTPDATE:time}] "%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}""
输出:
{
"message" => " 121.40.205.143 [29/Aug/2016:12:36:32 +0800] "GET /favicon.ico HTTP/1.1" - 404 2319 "-" "Mozilla/5.0 (Linux; Android 5.1.1; vivo X6S A Build/LMY47V) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/37.0.0.0 Mobile MQQBrowser/6.2 TBS/036558 Safari/537.36 MicroMessenger/6.3.25.861 NetType/WIFI Language/zh_CN" 0.000 -",
"@version" => "1",
"@timestamp" => "2016-08-29T04:39:16.608Z",
"path" => "/rsyslog/data/nginx/uat/nginx_access01_log.2016-08-29",
"host" => "0.0.0.0",
"type" => "uat_nginx_access",
"clientip" => "121.40.205.143",
"time" => "29/Aug/2016:12:36:32 +0800",
"verb" => "GET",
"request" => "/favicon.ico",
"httpversion" => "1.1"
}
此时grok 能正常匹配:
"message" => "%{IPORHOST:clientip} [%{HTTPDATE:time}] "%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}"
%{NUMBER:http_status_code} %{NUMBER:bytes} "(?<http_referer>S+)" "(?<http_user_agent>S+)" "(?<http_x_forwarded_for>S+)""
log_format main '$http_host $server_addr $remote_addr [$time_local] "$request" '
'$request_body $status $body_bytes_sent "$http_referer" "$http_user_agent" '
'$request_time $upstream_response_time';
继续加；
log_format main '$remote_addr [$time_local] "$request"'
'$status $body_bytes_sent';
日志格式:
121.40.205.143 [29/Aug/2016:12:51:18 +0800] "GET /resources/plugins/artDialog/ui-dialog.css HTTP/1.1"304 0
121.40.205.143 [29/Aug/2016:12:51:18 +0800] "GET /wechat/images/account/icons.7a340e21.png HTTP/1.1"304 0
121.40.205.143 [29/Aug/2016:12:51:18 +0800] "GET /wechat/images/nav-icon.44c2022c.png?v=1 HTTP/1.1"304 0
121.40.205.143 [29/Aug/2016:12:51:19 +0800] "GET /favicon.ico HTTP/1.1"404 2319
121.40.205.143 [29/Aug/2016:12:51:19 +0800] "GET /favicon.ico HTTP/1.1"404 2319
121.40.205.143 [29/Aug/2016:12:52:25 +0800] "GET /favicon.ico HTTP/1.1"404 2319
121.40.205.143 [29/Aug/2016:12:52:25 +0800] "GET /favicon.ico HTTP/1.1"404 2319
121.40.205.143 [29/Aug/2016:12:53:28 +0800] "GET /favicon.ico HTTP/1.1"404 2319
filter {
grok {
match=> {
"message" =>"%{IPORHOST:clientip} [%{HTTPDATE:time}] "%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}"%{NUMBER:http_status_code} %{NUMBER:bytes}"
}
logstash 输出:
Pipeline main started
{
"message" => " 121.40.205.143 [29/Aug/2016:12:56:10 +0800] "GET /favicon.ico HTTP/1.1"404 2319",
"@version" => "1",
"@timestamp" => "2016-08-29T04:58:54.908Z",
"path" => "/rsyslog/data/nginx/uat/nginx_access01_log.2016-08-29",
"host" => "0.0.0.0",
"type" => "uat_nginx_access",
"clientip" => "121.40.205.143",
"time" => "29/Aug/2016:12:56:10 +0800",
"verb" => "GET",
"request" => "/favicon.ico",
"httpversion" => "1.1",
"http_status_code" => "404",
"bytes" => "2319"
}
继续；
121.40.205.143 [29/Aug/2016:13:00:16 +0800] "GET /favicon.ico HTTP/1.1"404 2319 "-"
121.40.205.143 [29/Aug/2016:13:00:22 +0800] "GET /favicon.ico HTTP/1.1"404 2319 "-"
121.40.205.143 [29/Aug/2016:13:00:30 +0800] "GET /favicon.ico HTTP/1.1"404 2319 "-"
121.40.205.143 [29/Aug/2016:13:00:32 +0800] "GET /wechat/login.html HTTP/1.1"304 0 "https://uatest.winfae.com/wechat/account.html"
121.40.205.143 [29/Aug/2016:13:00:32 +0800] "GET /wechat/images/login/icon_01.6e839367.png HTTP/1.1"304 0 "https://uatest.winfae.com/wechat/css/wechat.2a00a782.css"
121.40.205.143 [29/Aug/2016:13:00:32 +0800] "GET /wechat/images/login/icon_02.5065faba.png HTTP/1.1"304 0 "https://uatest.winfae.com/wechat/css/wechat.2a00a782.css"
121.40.205.143 [29/Aug/2016:13:00:32 +0800] "GET /resources/plugins/jquery/jquery.md5.js?v=1 HTTP/1.1"304 0 "https://uatest.winfae.com/wechat/login.html"
121.40.205.143 [29/Aug/2016:13:00:32 +0800] "GET /wechat/js/libs/dialog-min.88247f5e.js?v=1 HTTP/1.1"304 0 "https://uatest.winfae.com/wechat/login.html"
121.40.205.143 [29/Aug/2016:13:00:32 +0800] "GET /wechat/js/login.a87fbd64.js HTTP/1.1"304 0 "https://uatest.winfae.com/wechat/login.html"
{
"message" => " 121.40.205.143 [29/Aug/2016:13:05:24 +0800] "GET /wechat/account_balance.html HTTP/1.1"200 3059 "https://uatest.winfae.com/wechat/account.html"
"Mozilla/5.0 (Linux; Android 5.1.1; vivo X6S A Build/LMY47V) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/37.0.0.0 Mobile MQQBrowser/6.2 TBS/036558 Safari/537.36 MicroMessenger/6.3.25.861 NetType/WIFI Language/zh_CN"",
121.40.205.143 [29/Aug/2016:13:05:24 +0800] "GET /wechat/account_balance.html HTTP/1.1"200 3059 "https://uatest.winfae.com/wechat/account.html" "Mozilla/5.0 (Linux; Android 5.1.1; vivo X6S A Build/LMY47V) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/37.0.0.0 Mobile MQQBrowser/6.2 TBS/036558 Safari/537.36 MicroMessenger/6.3.25.861 NetType/WIFI Language/zh_CN"
121.40.205.143 [29/Aug/2016:13:05:45 +0800] "GET /wechat/home.html?useragent=android_h5_zjcap&apiver=2 HTTP/1.1"200 11601 "-" "okhttp/2.6.0"
{
"message" => " 121.40.205.143 [29/Aug/2016:13:13:11 +0800] "GET /wechat/js/regain.431efde9.js HTTP/1.1"304 0 "https://uatest.winfae.com/wechat/regain.html" "Mozilla/5.0 (Linux; Android 5.1.1; vivo X6S A Build/LMY47V) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/37.0.0.0 Mobile MQQBrowser/6.2 TBS/036558 Safari/537.36 MicroMessenger/6.3.25.861 NetType/WIFI Language/zh_CN"",
"@version" => "1",
"@timestamp" => "2016-08-29T05:15:55.609Z",
"path" => "/rsyslog/data/nginx/uat/nginx_access01_log.2016-08-29",
"host" => "0.0.0.0",
"type" => "uat_nginx_access",
"clientip" => "121.40.205.143",
"time" => "29/Aug/2016:13:13:11 +0800",
"verb" => "GET",
"request" => "/wechat/js/regain.431efde9.js",
"httpversion" => "1.1",
"http_status_code" => "304",
"bytes" => "0",
"http_referer" => "https://uatest.winfae.com/wechat/regain.html"
S+ 和 [^
f]+ 语法一样 非空格
my $str=" begin 123.456 end ";
if ($str =~/(?<request_time>d+.d+)/)
{
my ($request_time) = ($+{request_time});
print $request_time."
";};
zjtest7-frontend:/root/0825# perl a1.pl
123.456
"http_referer" => "https://uatest.winfae.com/wechat/regain.html"
"(?<http_referer>S+)"
"(?<http_user_agent>S+)"
"Mozilla/5.0 (Linux; Android 5.1.1; vivo X6S A Build/LMY47V) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/37.0.0.0 Mobile MQQBrowser/6.2 TBS/036558 Safari/537.36 MicroMessenger/6.3.25.861 NetType/WIFI Language/zh_CN"
"message" => " 121.40.205.143 [29/Aug/2016:13:54:08 +0800] "GET /resources/plugins/artDialog/ui-dialog.css HTTP/1.1"200 9985 "https://uatest.winfae.com/wechat/home.html?useragent=ios_h5_zjcap&apiver=2&WKWebView=1" "ios_h5_zjcap"",
"@version" => "1",
"@timestamp" => "2016-08-29T05:56:53.217Z",
"path" => "/rsyslog/data/nginx/uat/nginx_access01_log.2016-08-29",
"host" => "0.0.0.0",
"type" => "uat_nginx_access",
"clientip" => "121.40.205.143",
"time" => "29/Aug/2016:13:54:08 +0800",
"verb" => "GET",
"request" => "/resources/plugins/artDialog/ui-dialog.css",
"httpversion" => "1.1",
"http_status_code" => "200",
"bytes" => "9985",
"http_referer" => "https://uatest.winfae.com/wechat/home.html?useragent=ios_h5_zjcap&apiver=2&WKWebView=1",
"http_user_agent" => "ios_h5_zjcap"
}
{
"message" => " 121.40.205.143 [29/Aug/2016:13:59:35 +0800] "GET /resources/js/toolbar.49fc367e.js?_v=${last.updated}&_=1472450673142 HTTP/1.1"200 1800 "https://uatest.winfae.com/products/productList.html" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.87 Safari/537.36"",
"@version" => "1",
"@timestamp" => "2016-08-29T06:02:18.775Z",
"path" => "/rsyslog/data/nginx/uat/nginx_access01_log.2016-08-29",
"host" => "0.0.0.0",
"type" => "uat_nginx_access",
"tags" => [
[0] "_grokparsefailure"
]
}