zoukankan      html  css  js  c++  java
  • logstash 读取kafka output ES

    input {
      kafka{
            bootstrap_servers => ["18.3.10.53:9092,18.3.10.54:9092,19.3.10.55:9092,19.3.10.56:9092,19.3.10.57:9092,19.3.10.91:9092,19.3.10.92:9092,19.3.10.93:9092"]
            auto_offset_reset => "latest"
            consumer_threads => 5
            decorate_events => true
            topics => ["mips_monitor_log"]
            type => "mips_monitor_log"
          }
          kafka{
            bootstrap_servers => ["19.3.10.53:9092,19.3.10.54:9092,19.3.10.55:9092,19.3.100.56:9092,19.3.10.57:9092,19.3.10.91:9092,19.3.10.92:9092,19.3.10.93:9092"]
            auto_offset_reset => "latest"
            consumer_threads => 5
            decorate_events => true
            topics => ["mips_info_log"]
            type => "mips_info_log"
          }
    }
    
    filter {
        multiline {
                    pattern => "^d{4}-d{1,2}-d{1,2}sd{1,2}:d{1,2}:d{1,2}"
                    negate => true
                    what => "previous"
                            }
        mutate {
            #从kafka的key中获取数据并按照逗号切割
            split => ["[@metadata][kafka][key]", ","]
            add_field => {
                #将切割后的第一位数据放入自定义的“index”字段中
                "ip" => "%{[@metadata][kafka][key][0]}"
            }
        }
    
    
    }
    
    output {
     if [type]=="mi_info_log"{
         elasticsearch {
             user =>admin
             password =>xxxxx
             ssl =>true
             ssl_certificate_verification => false
             truststore =>"/cslc/dip002/elk_data/logstash-6.5.1/config/truststore.jks"
             truststore_password =>"1deadxxxxxxxxxxxxxx2"
             hosts=> ["19.3.10.91:9200","19.3.10.92:9200","19.3.10.93:9200"]
             index =>"info_log-%{+YYYY.MM.dd}"
         }
     }
     if [type]=="monitor_log"{
         elasticsearch {
             user =>admin
             password =>xxxxxx
             ssl =>true
             ssl_certificate_verification => false
             truststore =>"/cslc/dip002/elk_data/logstash-6.5.1/config/truststore.jks"
             truststore_password =>"1xxxxxxxxxxxxxxxxxxxxx"
             hosts=> ["19.3.10.91:9200","19.3.10.92:9200","19.3.10.93:9200"]
             index =>"monitor_log-%{+YYYY.MM.dd}"
         }
     }
    }
  • 相关阅读:
    SOJ 2785_Binary Partitions
    Codeforces Round #328 (Div. 2)
    C++ fill 和memset
    SOJ 2749_The Fewest Coins
    Codeforces Round #327 (Div. 2)
    TYVJ P1013 找啊找啊找GF Label:动态规划
    TYVJ 1014 乘法游戏
    TYVJ 1011 NOIP 2008&&NOIP 2000 传纸条&&方格取数 Label:多线程dp
    错误集合
    TYVJ P1038/P1039 忠诚 标签:线段树
  • 原文地址:https://www.cnblogs.com/students/p/14339490.html
Copyright © 2011-2022 走看看