input{http{port=>7474}}
filter{
grok{
match =>{
#"message" => "%{COMBINEDAPACHELOG}"
"message" => '%{IPORHOST:clientip} %{USER:ident} %{USER:auth} [%{HTTPDATE:[@metadata][timestamp]}] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-) %{QS:referrer} %{QS:agent}'
}
}
# mutate{
# copy => { "@timestamp" => "read_timestamp"}
# }
ruby {
code => "event.set('@read_timestamp',event.get('@timestamp'))"
}
# 20/May/2015:21:05:15 +0000
#date{
# match => ["timestamp","dd/MMM/yyyy:HH:mm:ss Z"]
#}
date{
match => ["[@metadata][timestamp]","dd/MMM/yyyy:HH:mm:ss Z"]
}
geoip{
source => "clientip"
fields => ["latitude","longitude","city_name","country_name","region_name"]
}
useragent{
source => "agent"
target => "useragent"
}
mutate{
convert => { "bytes" => "integer" }
}
mutate{
remove_field =>["headers","message"]
}
}
output{stdout{codec=>rubydebug}}
apache_logstash.conf
input {
stdin { }
}
filter {
grok {
match => {
"message" => '%{IPORHOST:clientip} %{USER:ident} %{USER:auth} [%{HTTPDATE:timestamp}] "%{WORD:verb} %{DATA:request} HTTP/%{NUMBER:httpversion}" %{NUMBER:response:int} (?:-|%{NUMBER:bytes:int}) %{QS:referrer} %{QS:agent}'
}
}
date {
match => [ "timestamp", "dd/MMM/YYYY:HH:mm:ss Z" ]
locale => en
}
geoip {
source => "clientip"
}
useragent {
source => "agent"
target => "useragent"
}
}
output {
stdout {
codec => dots {}
}
elasticsearch {
index => "apache_elastic_example"
template => "./apache_template.json"
template_name => "apache_elastic_example"
template_overwrite => true
}
}
input{
#http{
# port => 7474
#}
stdin{}
# file{
# path => "/Users/rockybean/Downloads/es/6.1/logstash-6.1.1/demo_data/apache_logs/apache_logs"
# start_position => "beginning"
# }
}
filter{
#mutate{add_field => {"[@metadata][debug]"=>true}}
grok{
match => {
"message" => '%{IPORHOST:clientip} %{USER:ident} %{USER:auth} [%{HTTPDATE:[@metadata][timestamp]}] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-) %{QS:referrer} %{QS:agent}'
}
}
ruby{
code => "event.set('@read_timestamp',event.get('@timestamp'))"
}
# 20/May/2015:21:05:56 +0000
date{
match => ["[@metadata][timestamp]","dd/MMM/yyyy:HH:mm:ss Z"]
}
mutate{
convert => {"bytes" => "integer"}
}
geoip{
source => "clientip"
fields => ["location","country_name","city_name","region_name"]
}
useragent{
source => "agent"
target => "useragent"
}
mutate{remove_field=>["headers"]}
mutate{
add_field=>{
"[@metadata][index]" => "apache_logs_%{+YYYY.MM}"
}
}
if "_grokparsefailure" in [tags] {
mutate{
replace=>{
"[@metadata][index]" => "apache_logs_failure_%{+YYYY.MM}"
}
}
}else{
mutate{remove_field=>["message"]}
}
}
output{
if [@metadata][debug]{
stdout{codec=>rubydebug{metadata=>true}}
}else{
stdout{codec=>dots}
elasticsearch{
index => "%{[@metadata][index]}"
document_type => "doc"
}
}
}