logstash 字段引用

字段引用:


10.168.255.134 [09/Oct/2016:15:28:52 +0800] "GET / HTTP/1.1" - 200 23388 "" "Mozilla/5.0 (Linux; U; Android 4.4.4; zh-cn; MX4 Pro Build/KTU84P) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30" 0.001 101.226.125.103

[elk@Vsftp logstash]$ cat logstash.conf 
input {
   stdin{}
   }
filter {
    grok {
        match =>[ 
             "message","%{IPORHOST:clientip} [%{HTTPDATE:time}] "%{WORD:verb} %{URIPATHPARAM:request}?.* HTTP/%{NUMBER:httpversion}" - %{NUMBER:http_status_code} %{NUMBER:bytes} "(?<http_referer>S+)" "(?<http_user_agent>(S+s+)*S+)" (%{BASE16FLOAT:request_time}) (%{IPORHOST:http_x_forwarded_for}|-)", 
             "message" , "%{IPORHOST:clientip} [%{HTTPDATE:time}] "%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}" - %{NUMBER:http_status_code} %{NUMBER:bytes} "(?<http_referer>S+)" "(?<http_user_agent>(S+s+)*S+)" (%{BASE16FLOAT:request_time}) (%{IPORHOST:http_x_forwarded_for}|-)",
             "message","%{IPORHOST:clientip} [%{HTTPDATE:time}] "%{WORD:verb} (?<http_url>S+)s+HTTP/%{NUMBER:httpversion}"s+-s+%{NUMBER:http_status_code}s+%{NUMBER:bytes}s+"-"s+"(?<http_user_agent>(S+))"s+(%{BASE16FLOAT:request_time})s+(%{IPORHOST:http_x_forwarded_for}|-)",
             "message","%{IPORHOST:clientip} [%{HTTPDATE:time}] "%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}" - %{NUMBER:http_status_code} %{NUMBER:bytes} "" "(?<http_user_agent>(S+s+)*S+)" (%{BASE16FLOAT:request_time}) (%{IPORHOST:http_x_forwarded_for}|-)"
             
        ]
    }
}

output {
        stdout {
                        codec => rubydebug
                } 
}



[elk@Vsftp logstash]$ logstash -f logstash.conf 
Settings: Default pipeline workers: 4
Pipeline main started
10.168.255.134 [09/Oct/2016:15:28:52 +0800] "GET / HTTP/1.1" - 200 23388 "" "Mozilla/5.0 (Linux; U; Android 4.4.4; zh-cn; MX4 Pro Build/KTU84P) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30" 0.001 101.226.125.103
{
                 "message" => "10.168.255.134 [09/Oct/2016:15:28:52 +0800] "GET / HTTP/1.1" - 200 23388 "" "Mozilla/5.0 (Linux; U; Android 4.4.4; zh-cn; MX4 Pro Build/KTU84P) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30" 0.001 101.226.125.103",
                "@version" => "1",
              "@timestamp" => "2017-02-08T01:39:50.650Z",
                    "host" => "Vsftp",
                "clientip" => "10.168.255.134",
                    "time" => "09/Oct/2016:15:28:52 +0800",
                    "verb" => "GET",
                 "request" => "/",
             "httpversion" => "1.1",
        "http_status_code" => "200",
                   "bytes" => "23388",
         "http_user_agent" => "Mozilla/5.0 (Linux; U; Android 4.4.4; zh-cn; MX4 Pro Build/KTU84P) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30",
            "request_time" => "0.001",
    "http_x_forwarded_for" => "101.226.125.103"
}



[elk@Vsftp logstash]$ cat logstash.conf 
input {
   stdin{}
   }
filter {
    grok {
        match =>[ 
             "message","%{IPORHOST:clientip} [%{HTTPDATE:time}] "%{WORD:verb} %{URIPATHPARAM:request}?.* HTTP/%{NUMBER:httpversion}" - %{NUMBER:http_status_code} %{NUMBER:bytes} "(?<http_referer>S+)" "(?<http_user_agent>(S+s+)*S+)" (%{BASE16FLOAT:request_time}) (%{IPORHOST:http_x_forwarded_for}|-)", 
             "message" , "%{IPORHOST:clientip} [%{HTTPDATE:time}] "%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}" - %{NUMBER:http_status_code} %{NUMBER:bytes} "(?<http_referer>S+)" "(?<http_user_agent>(S+s+)*S+)" (%{BASE16FLOAT:request_time}) (%{IPORHOST:http_x_forwarded_for}|-)",
             "message","%{IPORHOST:clientip} [%{HTTPDATE:time}] "%{WORD:verb} (?<http_url>S+)s+HTTP/%{NUMBER:httpversion}"s+-s+%{NUMBER:http_status_code}s+%{NUMBER:bytes}s+"-"s+"(?<http_user_agent>(S+))"s+(%{BASE16FLOAT:request_time})s+(%{IPORHOST:http_x_forwarded_for}|-)",
             "message","%{IPORHOST:clientip} [%{HTTPDATE:time}] "%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}" - %{NUMBER:http_status_code} %{NUMBER:bytes} "" "(?<http_user_agent>(S+s+)*S+)" (%{BASE16FLOAT:request_time}) (%{IPORHOST:http_x_forwarded_for}|-)"
             
        ]
    }
geoip {
                        source => "http_x_forwarded_for"
                        target => "geoip"
                        database => "/usr/local/logstash-2.3.4/etc/GeoLiteCity.dat"
                        add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
                        add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}"  ]
                }
}

output {
        stdout {
                        codec => rubydebug
                } 
}

[elk@Vsftp logstash]$ logstash -f logstash.conf 
Settings: Default pipeline workers: 4
Pipeline main started
10.168.255.134 [09/Oct/2016:15:28:52 +0800] "GET / HTTP/1.1" - 200 23388 "" "Mozilla/5.0 (Linux; U; Android 4.4.4; zh-cn; MX4 Pro Build/KTU84P) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30" 0.001 101.226.125.103
{
                 "message" => "10.168.255.134 [09/Oct/2016:15:28:52 +0800] "GET / HTTP/1.1" - 200 23388 "" "Mozilla/5.0 (Linux; U; Android 4.4.4; zh-cn; MX4 Pro Build/KTU84P) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30" 0.001 101.226.125.103",
                "@version" => "1",
              "@timestamp" => "2017-02-08T01:42:33.645Z",
                    "host" => "Vsftp",
                "clientip" => "10.168.255.134",
                    "time" => "09/Oct/2016:15:28:52 +0800",
                    "verb" => "GET",
                 "request" => "/",
             "httpversion" => "1.1",
        "http_status_code" => "200",
                   "bytes" => "23388",
         "http_user_agent" => "Mozilla/5.0 (Linux; U; Android 4.4.4; zh-cn; MX4 Pro Build/KTU84P) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30",
            "request_time" => "0.001",
    "http_x_forwarded_for" => "101.226.125.103",
                   "geoip" => {
                      "ip" => "101.226.125.103",
           "country_code2" => "CN",
           "country_code3" => "CHN",
            "country_name" => "China",
          "continent_code" => "AS",
             "region_name" => "23",
               "city_name" => "Shanghai",
                "latitude" => 31.045600000000007,
               "longitude" => 121.3997,
                "timezone" => "Asia/Shanghai",
        "real_region_name" => "Shanghai",
                "location" => [
            [0] 121.3997,
            [1] 31.045600000000007
        ],
             "coordinates" => [
            [0] 121.3997,
            [1] 31.045600000000007
        ]
    }
}



字段引用

字段引用是Logstash::Event 对象的属性,我们之前提过事件就像一个哈希一样,所以你可以想象字段就像一个键值对


如果你想在Logstash 配置中使用字段的值,只需把字段的名字写在中括号[]里就行了,这就叫字段引用
[elk@Vsftp logstash]$ cat logstash.conf 
input {
   stdin{}
   }
filter {
    grok {
        match =>[ 
             "message","%{IPORHOST:clientip} [%{HTTPDATE:time}] "%{WORD:verb} %{URIPATHPARAM:request}?.* HTTP/%{NUMBER:httpversion}" - %{NUMBER:http_status_code} %{NUMBER:bytes} "(?<http_referer>S+)" "(?<http_user_agent>(S+s+)*S+)" (%{BASE16FLOAT:request_time}) (%{IPORHOST:http_x_forwarded_for}|-)", 
             "message" , "%{IPORHOST:clientip} [%{HTTPDATE:time}] "%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}" - %{NUMBER:http_status_code} %{NUMBER:bytes} "(?<http_referer>S+)" "(?<http_user_agent>(S+s+)*S+)" (%{BASE16FLOAT:request_time}) (%{IPORHOST:http_x_forwarded_for}|-)",
             "message","%{IPORHOST:clientip} [%{HTTPDATE:time}] "%{WORD:verb} (?<http_url>S+)s+HTTP/%{NUMBER:httpversion}"s+-s+%{NUMBER:http_status_code}s+%{NUMBER:bytes}s+"-"s+"(?<http_user_agent>(S+))"s+(%{BASE16FLOAT:request_time})s+(%{IPORHOST:http_x_forwarded_for}|-)",
             "message","%{IPORHOST:clientip} [%{HTTPDATE:time}] "%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}" - %{NUMBER:http_status_code} %{NUMBER:bytes} "" "(?<http_user_agent>(S+s+)*S+)" (%{BASE16FLOAT:request_time}) (%{IPORHOST:http_x_forwarded_for}|-)"
             
        ]
    }
geoip {
                        source => "http_x_forwarded_for"
                        target => "geoip"
                        database => "/usr/local/logstash-2.3.4/etc/GeoLiteCity.dat"
                        add_field => [ "aaaaaa", "%{[geoip][location][0]}" ]
                        add_field => [ "bbbbbb", "%{[geoip][location][1]}" ]
                }
}

output {
        stdout {
                        codec => rubydebug
                } 
}


[elk@Vsftp logstash]$ logstash -f logstash.conf 
Settings: Default pipeline workers: 4
Pipeline main started
10.168.255.134 [09/Oct/2016:15:28:52 +0800] "GET / HTTP/1.1" - 200 23388 "" "Mozilla/5.0 (Linux; U; Android 4.4.4; zh-cn; MX4 Pro Build/KTU84P) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30" 0.001 101.226.125.103
{
                 "message" => "10.168.255.134 [09/Oct/2016:15:28:52 +0800] "GET / HTTP/1.1" - 200 23388 "" "Mozilla/5.0 (Linux; U; Android 4.4.4; zh-cn; MX4 Pro Build/KTU84P) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30" 0.001 101.226.125.103",
                "@version" => "1",
              "@timestamp" => "2017-02-08T01:47:32.656Z",
                    "host" => "Vsftp",
                "clientip" => "10.168.255.134",
                    "time" => "09/Oct/2016:15:28:52 +0800",
                    "verb" => "GET",
                 "request" => "/",
             "httpversion" => "1.1",
        "http_status_code" => "200",
                   "bytes" => "23388",
         "http_user_agent" => "Mozilla/5.0 (Linux; U; Android 4.4.4; zh-cn; MX4 Pro Build/KTU84P) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30",
            "request_time" => "0.001",
    "http_x_forwarded_for" => "101.226.125.103",
	
                   "geoip" => {
                      "ip" => "101.226.125.103",
           "country_code2" => "CN",
           "country_code3" => "CHN",
            "country_name" => "China",
          "continent_code" => "AS",
             "region_name" => "23",
               "city_name" => "Shanghai",
                "latitude" => 31.045600000000007,
               "longitude" => 121.3997,
                "timezone" => "Asia/Shanghai",
        "real_region_name" => "Shanghai",
                "location" => [
            [0] 121.3997,
            [1] 31.045600000000007
        ]
    },
	
	
                  "aaaaaa" => 121.3997,
                  "bbbbbb" => 31.045600000000007
}



变量值内插:
[elk@Vsftp logstash]$ cat logstash.conf 
input {
   stdin{}
   }
filter {
    grok {
        match =>[ 
             "message","%{IPORHOST:clientip} [%{HTTPDATE:time}] "%{WORD:verb} %{URIPATHPARAM:request}?.* HTTP/%{NUMBER:httpversion}" - %{NUMBER:http_status_code} %{NUMBER:bytes} "(?<http_referer>S+)" "(?<http_user_agent>(S+s+)*S+)" (%{BASE16FLOAT:request_time}) (%{IPORHOST:http_x_forwarded_for}|-)", 
             "message" , "%{IPORHOST:clientip} [%{HTTPDATE:time}] "%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}" - %{NUMBER:http_status_code} %{NUMBER:bytes} "(?<http_referer>S+)" "(?<http_user_agent>(S+s+)*S+)" (%{BASE16FLOAT:request_time}) (%{IPORHOST:http_x_forwarded_for}|-)",
             "message","%{IPORHOST:clientip} [%{HTTPDATE:time}] "%{WORD:verb} (?<http_url>S+)s+HTTP/%{NUMBER:httpversion}"s+-s+%{NUMBER:http_status_code}s+%{NUMBER:bytes}s+"-"s+"(?<http_user_agent>(S+))"s+(%{BASE16FLOAT:request_time})s+(%{IPORHOST:http_x_forwarded_for}|-)",
             "message","%{IPORHOST:clientip} [%{HTTPDATE:time}] "%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}" - %{NUMBER:http_status_code} %{NUMBER:bytes} "" "(?<http_user_agent>(S+s+)*S+)" (%{BASE16FLOAT:request_time}) (%{IPORHOST:http_x_forwarded_for}|-)"
             
        ]
    }
geoip {
                        source => "http_x_forwarded_for"
                        target => "geoip"
                        database => "/usr/local/logstash-2.3.4/etc/GeoLiteCity.dat"
                        add_field => [ "kkkkkkk", "[geoip][location][0]"]
                        add_field => [ "hhhhhhh", "[geoip][location][1]" ]
                }
}

output {
        stdout {
                        codec => rubydebug
                } 
}

[elk@Vsftp logstash]$ logstash -f logstash.conf 
Settings: Default pipeline workers: 4
Pipeline main started
10.168.255.134 [09/Oct/2016:15:28:52 +0800] "GET / HTTP/1.1" - 200 23388 "" "Mozilla/5.0 (Linux; U; Android 4.4.4; zh-cn; MX4 Pro Build/KTU84P) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30" 0.001 101.226.125.103
{
                 "message" => "10.168.255.134 [09/Oct/2016:15:28:52 +0800] "GET / HTTP/1.1" - 200 23388 "" "Mozilla/5.0 (Linux; U; Android 4.4.4; zh-cn; MX4 Pro Build/KTU84P) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30" 0.001 101.226.125.103",
                "@version" => "1",
              "@timestamp" => "2017-02-08T01:49:49.034Z",
                    "host" => "Vsftp",
                "clientip" => "10.168.255.134",
                    "time" => "09/Oct/2016:15:28:52 +0800",
                    "verb" => "GET",
                 "request" => "/",
             "httpversion" => "1.1",
        "http_status_code" => "200",
                   "bytes" => "23388",
         "http_user_agent" => "Mozilla/5.0 (Linux; U; Android 4.4.4; zh-cn; MX4 Pro Build/KTU84P) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30",
            "request_time" => "0.001",
    "http_x_forwarded_for" => "101.226.125.103",
                   "geoip" => {
                      "ip" => "101.226.125.103",
           "country_code2" => "CN",
           "country_code3" => "CHN",
            "country_name" => "China",
          "continent_code" => "AS",
             "region_name" => "23",
               "city_name" => "Shanghai",
                "latitude" => 31.045600000000007,
               "longitude" => 121.3997,
                "timezone" => "Asia/Shanghai",
        "real_region_name" => "Shanghai",
                "location" => [
            [0] 121.3997,
            [1] 31.045600000000007
        ]
    },
                 "kkkkkkk" => "[geoip][location][0]",
                 "hhhhhhh" => "[geoip][location][1]"
				 
				 
	必须使用        add_field => [ "aaaaaa", "%{[geoip][location][0]}" ]
                        add_field => [ "bbbbbb", "%{[geoip][location][1]}" ]
}