zoukankan      html  css  js  c++  java
  • logstash 读取kafka output ES

    input {
      kafka{
            bootstrap_servers => ["18.3.10.53:9092,18.3.10.54:9092,19.3.10.55:9092,19.3.10.56:9092,19.3.10.57:9092,19.3.10.91:9092,19.3.10.92:9092,19.3.10.93:9092"]
            auto_offset_reset => "latest"
            consumer_threads => 5
            decorate_events => true
            topics => ["mips_monitor_log"]
            type => "mips_monitor_log"
          }
          kafka{
            bootstrap_servers => ["19.3.10.53:9092,19.3.10.54:9092,19.3.10.55:9092,19.3.100.56:9092,19.3.10.57:9092,19.3.10.91:9092,19.3.10.92:9092,19.3.10.93:9092"]
            auto_offset_reset => "latest"
            consumer_threads => 5
            decorate_events => true
            topics => ["mips_info_log"]
            type => "mips_info_log"
          }
    }
    
    filter {
        multiline {
                    pattern => "^d{4}-d{1,2}-d{1,2}sd{1,2}:d{1,2}:d{1,2}"
                    negate => true
                    what => "previous"
                            }
        mutate {
            #从kafka的key中获取数据并按照逗号切割
            split => ["[@metadata][kafka][key]", ","]
            add_field => {
                #将切割后的第一位数据放入自定义的“index”字段中
                "ip" => "%{[@metadata][kafka][key][0]}"
            }
        }
    
    
    }
    
    output {
     if [type]=="mi_info_log"{
         elasticsearch {
             user =>admin
             password =>xxxxx
             ssl =>true
             ssl_certificate_verification => false
             truststore =>"/cslc/dip002/elk_data/logstash-6.5.1/config/truststore.jks"
             truststore_password =>"1deadxxxxxxxxxxxxxx2"
             hosts=> ["19.3.10.91:9200","19.3.10.92:9200","19.3.10.93:9200"]
             index =>"info_log-%{+YYYY.MM.dd}"
         }
     }
     if [type]=="monitor_log"{
         elasticsearch {
             user =>admin
             password =>xxxxxx
             ssl =>true
             ssl_certificate_verification => false
             truststore =>"/cslc/dip002/elk_data/logstash-6.5.1/config/truststore.jks"
             truststore_password =>"1xxxxxxxxxxxxxxxxxxxxx"
             hosts=> ["19.3.10.91:9200","19.3.10.92:9200","19.3.10.93:9200"]
             index =>"monitor_log-%{+YYYY.MM.dd}"
         }
     }
    }
  • 相关阅读:
    Docker从12升级到17ce
    镜像清理和删除
    flask-session 在redis中存储session
    linux后台运行python程序 nohup
    flask 自动切换环境
    Linux SSH登录很慢的解决方法
    docker-compose docker启动工具,容器互联
    为什么企业需要IT资产管理
    sql 中取整,四舍五入取整,向下取整,向上取整。
    sqlalchemy 获取表结构。
  • 原文地址:https://www.cnblogs.com/students/p/14339490.html
Copyright © 2011-2022 走看看