zoukankan      html  css  js  c++  java
  • logstash 读取kafka output ES

    input {
      kafka{
            bootstrap_servers => ["18.3.10.53:9092,18.3.10.54:9092,19.3.10.55:9092,19.3.10.56:9092,19.3.10.57:9092,19.3.10.91:9092,19.3.10.92:9092,19.3.10.93:9092"]
            auto_offset_reset => "latest"
            consumer_threads => 5
            decorate_events => true
            topics => ["mips_monitor_log"]
            type => "mips_monitor_log"
          }
          kafka{
            bootstrap_servers => ["19.3.10.53:9092,19.3.10.54:9092,19.3.10.55:9092,19.3.100.56:9092,19.3.10.57:9092,19.3.10.91:9092,19.3.10.92:9092,19.3.10.93:9092"]
            auto_offset_reset => "latest"
            consumer_threads => 5
            decorate_events => true
            topics => ["mips_info_log"]
            type => "mips_info_log"
          }
    }
    
    filter {
        multiline {
                    pattern => "^d{4}-d{1,2}-d{1,2}sd{1,2}:d{1,2}:d{1,2}"
                    negate => true
                    what => "previous"
                            }
        mutate {
            #从kafka的key中获取数据并按照逗号切割
            split => ["[@metadata][kafka][key]", ","]
            add_field => {
                #将切割后的第一位数据放入自定义的“index”字段中
                "ip" => "%{[@metadata][kafka][key][0]}"
            }
        }
    
    
    }
    
    output {
     if [type]=="mi_info_log"{
         elasticsearch {
             user =>admin
             password =>xxxxx
             ssl =>true
             ssl_certificate_verification => false
             truststore =>"/cslc/dip002/elk_data/logstash-6.5.1/config/truststore.jks"
             truststore_password =>"1deadxxxxxxxxxxxxxx2"
             hosts=> ["19.3.10.91:9200","19.3.10.92:9200","19.3.10.93:9200"]
             index =>"info_log-%{+YYYY.MM.dd}"
         }
     }
     if [type]=="monitor_log"{
         elasticsearch {
             user =>admin
             password =>xxxxxx
             ssl =>true
             ssl_certificate_verification => false
             truststore =>"/cslc/dip002/elk_data/logstash-6.5.1/config/truststore.jks"
             truststore_password =>"1xxxxxxxxxxxxxxxxxxxxx"
             hosts=> ["19.3.10.91:9200","19.3.10.92:9200","19.3.10.93:9200"]
             index =>"monitor_log-%{+YYYY.MM.dd}"
         }
     }
    }
  • 相关阅读:
    大二下学期团队项目(app端web请求)
    大二下每周总结
    大二下学期团队项目(手机端分类查询前端)
    Windows 2008 R2 64位下安装Oracle 10.2.0.5
    Win7 32位下安装Oracle 10g
    SQL Developer安装使用教程
    Oracle使用DBCA建立数据库实例
    史上最全Oracle安装配置图文教程,含TNS配置
    Oracle9i完美安装、配置及操作
    安装MySQL启动服务时报错解决办法,错误代码1045
  • 原文地址:https://www.cnblogs.com/students/p/14339490.html
Copyright © 2011-2022 走看看