zoukankan      html  css  js  c++  java
  • logstash 读取kafka output ES

    input {
      kafka{
            bootstrap_servers => ["18.3.10.53:9092,18.3.10.54:9092,19.3.10.55:9092,19.3.10.56:9092,19.3.10.57:9092,19.3.10.91:9092,19.3.10.92:9092,19.3.10.93:9092"]
            auto_offset_reset => "latest"
            consumer_threads => 5
            decorate_events => true
            topics => ["mips_monitor_log"]
            type => "mips_monitor_log"
          }
          kafka{
            bootstrap_servers => ["19.3.10.53:9092,19.3.10.54:9092,19.3.10.55:9092,19.3.100.56:9092,19.3.10.57:9092,19.3.10.91:9092,19.3.10.92:9092,19.3.10.93:9092"]
            auto_offset_reset => "latest"
            consumer_threads => 5
            decorate_events => true
            topics => ["mips_info_log"]
            type => "mips_info_log"
          }
    }
    
    filter {
        multiline {
                    pattern => "^d{4}-d{1,2}-d{1,2}sd{1,2}:d{1,2}:d{1,2}"
                    negate => true
                    what => "previous"
                            }
        mutate {
            #从kafka的key中获取数据并按照逗号切割
            split => ["[@metadata][kafka][key]", ","]
            add_field => {
                #将切割后的第一位数据放入自定义的“index”字段中
                "ip" => "%{[@metadata][kafka][key][0]}"
            }
        }
    
    
    }
    
    output {
     if [type]=="mi_info_log"{
         elasticsearch {
             user =>admin
             password =>xxxxx
             ssl =>true
             ssl_certificate_verification => false
             truststore =>"/cslc/dip002/elk_data/logstash-6.5.1/config/truststore.jks"
             truststore_password =>"1deadxxxxxxxxxxxxxx2"
             hosts=> ["19.3.10.91:9200","19.3.10.92:9200","19.3.10.93:9200"]
             index =>"info_log-%{+YYYY.MM.dd}"
         }
     }
     if [type]=="monitor_log"{
         elasticsearch {
             user =>admin
             password =>xxxxxx
             ssl =>true
             ssl_certificate_verification => false
             truststore =>"/cslc/dip002/elk_data/logstash-6.5.1/config/truststore.jks"
             truststore_password =>"1xxxxxxxxxxxxxxxxxxxxx"
             hosts=> ["19.3.10.91:9200","19.3.10.92:9200","19.3.10.93:9200"]
             index =>"monitor_log-%{+YYYY.MM.dd}"
         }
     }
    }
  • 相关阅读:
    react组件之间传值方式
    html url 传递锚点并添加参数
    Spring Boot 构建WAR包
    Spring Boot Actuator 的使用
    Spring boot的启动加载原理
    intellij idea resin容器部署web工程
    Mybatis Mapper之见解
    踩坑----数据库阻塞
    redis缓存与数据库的记录不一致造成的问题.(乐观锁)
    H5中popstate事件的诡异行为
  • 原文地址:https://www.cnblogs.com/students/p/14339490.html
Copyright © 2011-2022 走看看