zoukankan      html  css  js  c++  java
  • logstash-kafka2elastic

    nohup bin/logstash -f config/news_ai_result.conf  > news_ai_result.log &
    --------------------------------------------------------------------------------------------------------
    [2018-08-15 15:56:54 hadoop@ecsc2901 /apps/logstash-ai/config]$ vi news_ai_result.conf 
    input {
            kafka {
                    bootstrap_servers => "172.17.129.101:9092,172.17.129.102:9092,172.17.129.103:9092,172.17.129.104:9092,172.17.129.105:9092,172.17.129.106:9092,172.17.129.107:9092,172.17.129.108:9092,172.17.129.109:9092"
                    topics => ["news_ai_result"]
                    group_id => "logstash_news_ai_result"
                    auto_commit_interval_ms => "5000"
                    auto_offset_reset => "earliest"
                    max_poll_records => "100"
            }
    }
    
    filter {
            json {
                    source => "message"
                    remove_field => "message"
                    remove_field => "contenthtml"
            }
            date {
                    match => ["extractdate", "yyyy-MM-dd HH:mm:ss"]
            }
    }
    output {
            #elasticsearch {
            #        hosts => "172.17.129.102:9200"
            #        index => "news_ai_result-%{+YYYY.MM.dd}"
            #        flush_size => 50000
            #}
    
            elasticsearch {
                    hosts => ["172.17.129.101:9200","172.17.129.102:9201","172.17.129.103:9202","172.17.129.104:9203","172.17.129.104:9204"]
                    index => "news_ai_result"
                    document_id => "%{rowkey}"
                    template_overwrite => true
                    template => "/apps/mapping/news_ai_result.json"
                    template_name=> "index_news_ai_result"
            }
    }
    -------------------------------------------------------------------------------------------------
    [2018-08-15 15:57:34 hadoop@ecsc2901 /apps/logstash-ai/config]$ vi /apps/mapping/news_ai_result.json 
    
    {
            "template" : "index_news_ai_result*",
            "settings" : {
                    "index.number_of_shards" : 5,
                    "number_of_replicas" : 1,
                    "index.refresh_interval" : "1s"
            },
            "dynamic_date_formats":["yyyy-MM-dd HH:mm:ss"],
            "dynamic":false,
            "mappings": {
                    "logs": {
                            "properties": {
                                    "author":{"type":"string","index":"not_analyzed","store":true,"fielddata":{"format":"fst"}},
                                    "cleandate":{"type": "string","index": "not_analyzed", "store": true,"fielddata":{"format": "fst"}},
                                    "extractdate":{"type": "string","index": "not_analyzed", "store": true,"fielddata":{"format": "fst"}},
                                    "extracttype":{"type": "string","index": "not_analyzed", "store": true,"fielddata":{"format": "fst"}},
                                    "publictime":{"type": "string","index": "not_analyzed", "store": true,"fielddata":{"format": "fst"}},
                                    "source":{"type": "string","index": "not_analyzed", "store": true,"fielddata":{"format": "fst"}},
                                    "title":{"type": "string","index": "not_analyzed", "store": true,"fielddata":{"format": "fst"}},
                                    "dataflag": {"type": "string","index": "not_analyzed", "store": true,"fielddata":{"format": "fst"}},
                                    "downloaddate": {"type": "string","index": "not_analyzed", "store": true,"fielddata":{"format": "fst"}},
                                    "enumber": {"type": "string","index": "not_analyzed", "store": true,"fielddata": {"format": "fst"}},
                                    "extractflag": {"type": "string","index": "not_analyzed", "store":true,"fielddata": {"format": "fst"}},
                                    "rowkey": {"type": "string","index": "not_analyzed", "store": true, "fielddata": {"format": "fst"}},
                                    "url": {"type": "string","index": "not_analyzed", "store": true, "fielddata": {"format": "fst"}},
                                    "domain": {"type": "string","index": "not_analyzed", "store": true, "fielddata": {"format": "fst"}}
                            }
                    }
            }
    }

  • 相关阅读:
    使用RestTemplate进行服务调用的几种方式
    springmvc学习指南 之---第32篇 mybatis 嵌套的处理
    springmvc学习指南 之---第31篇 使用墨客进行测试报错
    springmvc学习指南 之---第30篇 异常的全局处理
    Effective Java 阅读笔记--之(一) 建造者模式(Builder)
    使用mybatis-generator.xml 生成PO 对象
    springmvc学习指南 之---第29篇 springmvc 返回json对象, 不想创建类的前提下
    springmvc学习指南 之---第28篇 springmvc的controller 如何解析视图view? 如何解析json,html,jsp?
    springmvc学习指南 之---第27篇 spring如何实现servlet3.0无web.xml 配置servlet对象的
    springmvc学习指南 之---第26篇 在idea中如何debug跟踪到tomcat内部代码
  • 原文地址:https://www.cnblogs.com/zhaohz/p/12117169.html
Copyright © 2011-2022 走看看