简单记录一下logstash的语法

本文将一些语法简单的用列子展示出来
1.json

input {
    kafka{
        bootstrap_servers => ["10.0.0.11:9092,10.0.0.12:9092,10.0.0.13:9092"]
        group_id => "es"
        topics => ["dhc"]
        codec => json 
}
}
filter {
if "dhc" in [tags]{
   json {
     source => "message"
  }
if "." not in [value]{
        mutate {
        add_field => { "value_int" => "%{[value]}" }
        }
        }
mutate { 
convert => { "value_int" => "integer"  }
 }
}

}

output {
#   stdout {}
   if "dhc" in [tags] {
      elasticsearch {
        hosts => ["10.0.0.11:9200","10.0.0.12:9200","10.0.0.13:9200"] 
        manage_template => false
        index => "dhc-%{+yyyy.MM.dd}"
        user => "elastic"
        password => "111111"
      }
    }
}

2. 非json+调整时间（东八区）+按照日志时间生成+修改字段属性+截取字段长度

懒得标注释了，太长

input {

kafka{
                bootstrap_servers => ["10.100.15.23:9092,10.100.15.24:9092,10.100.15.25:9092"]
                group_id => "test"
                auto_offset_reset => "earliest"
                consumer_threads => "5"
                decorate_events => "false"
                topics => ["kafka"]
                type => "bbs_log"
                codec => json

        }
}


filter {

  if "km" in [tags]{
  grok {
    match => { "message" => "%{TIMESTAMP_ISO8601:times}s*S+<(?<MODULE>(.*))>sS+:<(?<level>(.*))\,(?<SNO>(.*))>sS+%{GREEDYDATA:message_value}" }
  }
  date {
        match => ["times", "ISO8601"]
        remove_field => "times"
    }
  mutate{
    convert => { "level" => "integer" }
  }
}
  if "shbkuc" in [tags]{
  grok {
    match => { "message" => "%{TIMESTAMP_ISO8601:times}s*S+<(?<MODULE>(.*))>sS+:<(?<level>(.?))\,(?<SNO>(.*))>sS+%{GREEDYDATA:message_value}" }
  }
  date {
        match => ["times", "ISO8601"]
        remove_field => "times"
    }
}

  if "time" in [tags]{
  grok {
         match => { "message" => "%{TIMESTAMP_ISO8601:times}s%{USER:info}s(?<SNO>(.*))s(?<excute_time>(.*))s[(?<message_value>(.{0,240}))]" }
}
  date {
        match => ["times", "ISO8601"]
        remove_field => "times"
    }

if "(null)" not in [excute_time]{
        mutate {
        add_field => { "cuttime" => "%{[excute_time]}" }
        }
        }
mutate { convert => { "cuttime" => "integer"  } }
}


if "test" in [tags]{   
   grok {
	 match => { "message" => "%{TIMESTAMP_ISO8601:DATE_time}s*%{USER:server_name}S+s*%{INT:level}\,%{BASE16NUM:SNO}\,(?<excute_time>(.*));sS+%{GREEDYDATA:message_value}"
	}
        }
    }

  if "caserver" in [tags]{
  grok {
         match => { "message" => "%{TIMESTAMP_ISO8601:times}s*%{USER:server_name}S+s*%{INT:level}\,(?<SNO>(.*))\,(?<excute_time>(.*));sS+%{GREEDYDATA:message_value}" }
}
  date {
        match => ["times", "ISO8601"]
        remove_field => "times"
    }
  mutate{
    convert => { "level" => "integer" }
  }
if "(null)" not in [excute_time]{
	mutate {
#	add_field => { "twotime" => "%{excute_time}" }
	split => [ "excute_time","."]
	add_field => { "cuttime" => "%{[excute_time][0]}" } 
	add_field => { "cuttime_last" => "%{[excute_time][1]}" }
	}
#	mutate {
#	convert => { "cuttime" => "integer" }
#	}
	}
mutate { convert => { "cuttime" => "integer"  } }
}


  if "exuc" in [tags]{
  grok {
         match => { "message" => "%{TIMESTAMP_ISO8601:times}s%{USER:server_name}s*%{INT:level}s(?<SNO>(.*))s(?<excute_time>(.*))s[%{GREEDYDATA:message_value}]" }
}
  date {
        match => ["times", "ISO8601"]
        remove_field => "times"
    }
  mutate{
    convert => { "level" => "integer" }
  }
if "(null)" not in [excute_time]{
	mutate {
#	add_field => { "twotime" => "%{excute_time}" }
	split => [ "excute_time","."]
	add_field => { "cuttime" => "%{[excute_time][0]}" } 
	add_field => { "cuttime_last" => "%{[excute_time][1]}" }
	}
#	mutate {
#	convert => { "cuttime" => "integer" }
#	}
	}
mutate { convert => { "cuttime" => "integer"  } }
}


  if "json" in [tags]{
  grok {
    match => { "message" => "%{TIMESTAMP_ISO8601:times}s*S+<(?<MODULE>(.*))>sS+:<(?<lv_num>(.*))\,(?<lv_SNO>(.*))>sS+%{GREEDYDATA:message_value}" }
  }
  date {
        match => ["times", "ISO8601"]
       # locale => "en"
       # timezone => "+00:00"
        remove_field => "times"
    }
}
}
output {
#   stdout {}
   if "caserver" in [tags] {
      elasticsearch {
        hosts => ["10.100.15.23:9200","10.100.15.24:9200","10.100.15.25:9200"] 
        manage_template => false
        index => "caserver-%{+yyyy.MM}"
        user => "elastic"
        password => "elk_123"
      }
    }
}