elk 配置

docker 启动

参考:https://www.elastic.co/guide/en/elasticsearch/reference/5.1/docker.html#docker-cli-run-prod-mode

查询语句：https://n3xtchen.github.io/n3xtchen/elasticsearch/2017/07/05/elasticsearch-23-useful-query-example

mac系统配置参数：

screen ~/Library/Containers/com.docker.docker/Data/com.docker.driver.amd64-linux/tty
# or screen ~/Library/Containers/com.docker.docker/Data/vms/0/tty
# find ~/Library/Containers/com.docker.docker/Data/ -name 'tty'

sysctl -w vm.max_map_count=262144

Exit by Control-A Control-

删除部分container

增加一个行yam文件

docker-compose -f docker-compose-production.yml down -v  # 同时删除volum

批量操作

一般建议是1000-5000个文档，如果你的文档很大，可以适当减少队列，大小建议是5-15MB，默认不能超过100M

logstash

input file error path.data
删除 data/.lock 文件
doe logstash ./bin/logstash -f custom_file/event_app.conf --path.data=path_data
doe logstash ./bin/logstash --debug -f custom_file/event_app.conf --path.data=path_data
不停止logstash，更新配置文件 kill -HUP pid (pid 用 top 查看)

同步 mysql 到elasticsearch
把jdbc docker cp 到 /usr/share/logstash/logstash-core/lib/jars

同步mongo monstache

doe monstache monstache -f ./custome_files/mongo_es.toml -verbose

# 前提条件，mongo开启副本集，用户可以访问local数据库，

mongo-url = "mongodb://192.168.10.47:27017,192.168.10.47:27018/?replicaSet=my-mongo-rs"

elasticsearch-urls = ["http://192.168.10.47:9200"]
elasticsearch-user = ""
elasticsearch-password = ""

# 存储monstache元信息的数据库名称
config-database-name = "mongoache"
# monstache集群名称
cluster-name = "mongo-es"

# 删除策略  0: 删除  1: 弃用  2: 忽略删除动作
#delete-strategy = 0
#delete-index-pattern = "mydb"

# 如果需要全量同步多个集合, 该参数控制是否同时同步, 如果是1会一个一个同步
direct-read-concur = 1

# 需同步的集合
namespace-regex = '^jieloan.api_order_data'
# 需全量同步的集合, 第一次运行时必须打开, 以后运行时必须关闭, 否则每次都做全量同步
direct-read-namespaces = ["jieloan.api_order_data"]
# 需增量同步的集合, change-stream只有在3.6后才能使用
# change-stream-namespaces = ["jieloan.api_order_data"]
# mongo3.6之前的版本需开启, 3.6版本后添加了change-stream的功能
enable-oplog = true
# 全量同步时, 设置no cursor timeout
direct-read-no-timeout = true
# 全量同步时,把集合分成n个range, 由n个go routines去处理, 增加该参数会增加连接数和monstache的内存, 会提高吞吐量, -1则是使用一个cursor去迭代所有数据
direct-read-split-max = 8

# 如果设为true, 那么只做全量同步, 不同步更新
disable-change-events = false
# 是否同步数据库或集合删除事件
dropped-databases = true
dropped-collections = true

# 发送到es的请求失败了, 是否重试
elasticsearch-retry = false
# 连接es的连接池, 控制bulk的并发数量
elasticsearch-max-conns = 8

# 当文档数达到多少时, 发送一次bulk请求, 该参数一般不推荐使用
elasticsearch-max-docs = -1
# 当connection buffer达到多少字节时, 发送一次bulk, 默认8M, 该参数能有效提高性能, 增加该参数意味着消耗更多内存
#elasticsearch-max-bytes =

# 是否使用easy-json处理序列化
enable-easy-json = false
# 全量同步完后立刻退出, 该参数一般用于定时任务
exit-after-direct-reads = false
# 当收到失败的事件时, 是否立刻退出程序
fail-fast = false
gzip = true

# 开启该参数才能与mysql的数据一起index
index-as-update = true
index-oplog-time = false
# 是否在es中创建index来保存统计信息
index-stats = true

mongo-oplog-database-name = "local"
mongo-oplog-collection-name = "oplog.rs"
prune-invalid-json = true

# deprecated
replay = false
# 如果开启该参数, monstache会把已经处理的事件的事件写入mongodb, 如果monstache意外中断,那么下次启动时,会从该时间点同步, 如果设置cluster-name,该参数字段开启
resume = true
#resume-from-timestamp = 0

# use a custom resume strategy (tokens) instead of the default strategy (timestamps)
# tokens work with MongoDB API 3.6+ while timestamps work only with MongoDB API 4.0+
#resume-strategy = 0

enable-http-server = true
http-server-addr = "0.0.0.0:8080"

[[mapping]]
namespace = "jieloan.api_order_data"
index = "loan_apiorder"

搜索引擎 衡量 infomation retrieval

1. precision (查准率) - 尽可能返回较少的无关文档
   2.recall (查全率) - 尽量返回较多的相关文档
   3.ranking - 能否按照相关度进行排序

kibana 高亮highlight有[1000000]限制，且会减慢查询速度

关闭高亮显示 doc_table:highlight

• 在es同步大量数据时可以通过对索引增加分片来提升写的能力，之后到了稳定的增量阶段可以通过合并分片来提升查询性能。

• 可以考虑关闭动态索引

put _cluster/settings
{
	"persistent": {
		"action.auto_create_index": false
	}
}

通过模版设置白名单

put _cluster/settings
{
	"persistent": {
		"action.auto_create_index": "logstash-*, .kibana*"
	}
}

查询日志入ES

经纬度查询

GET loan_apiorder/_search
{
  "query": {
    "bool": {
      "must": [
        {
          "geo_distance":{
            "distance": "500m",
            "location":{
              "lat":31,
              "lon":121
            }
          }
        }
      ]
    }
  }
}

watcher

{
  "_index" : ".watches",
  "_type" : "_doc",
  "_id" : "test3",
  "_version" : 2954,
  "_seq_no" : 1452147,
  "_primary_term" : 10,
  "found" : true,
  "_source" : {
    "trigger" : {
      "schedule" : {
        "interval" : "1m"
      }
    },
    "input" : {
      "search" : {
        "request" : {
          "search_type" : "query_then_fetch",
          "indices" : [
            "email_sent*"
          ],
          "rest_total_hits_as_int" : true,
          "body" : {
            "size" : 0,
            "query" : {
              "bool" : {
                "must" : [
                  {
                    "range" : {
                      "datetime" : {
                        "gt" : "now-10m",
                        "lt" : "now"
                      }
                    }
                  },
                  {
                    "match" : {
                      "subject" : "micros"
                    }
                  }
                ]
              }
            }
          }
        }
      }
    },
    "condition" : {
      "compare" : {
        "ctx.payload.hits.total" : {
          "gte" : 100
        }
      }
    },
    "actions" : {
      "my_webhook" : {
        "throttle_period_in_millis" : 600000,
        "webhook" : {
          "scheme" : "http",
          "host" : "i1.celtgame.com",
          "port" : 80,
          "method" : "post",
          "path" : "/es_alert",
          "params" : { },
          "headers" : { },
          "body" : """
{
        "msgtype": "text",
        "text": {
            "content": "过去10分钟， 微服务报错: {{ctx.payload.hits.total}}条"
        }
   }
"""
        }
      },
      "index_payload" : {
        "transform" : {
          "script" : {
            "source" : "return [ 'scheduled_time' : ctx.trigger.scheduled_time , 'watch_id': ctx.watch_id , 'ts': System.currentTimeMillis()/1000 ]",
            "lang" : "painless"
          }
        },
        "index" : {
          "index" : "watch_alert"
        }
      }
    },
    "status" : {
      "state" : {
        "active" : true,
        "timestamp" : "2020-04-23T03:27:43.143Z"
      },
      "actions" : {
        "index_payload" : {
          "ack" : {
            "timestamp" : "2020-04-24T00:42:43.206Z",
            "state" : "awaits_successful_execution"
          },
          "last_execution" : {
            "timestamp" : "2020-04-24T00:41:43.159Z",
            "successful" : true
          },
          "last_successful_execution" : {
            "timestamp" : "2020-04-24T00:41:43.159Z",
            "successful" : true
          }
        },
        "my_webhook" : {
          "ack" : {
            "timestamp" : "2020-04-24T00:42:43.206Z",
            "state" : "awaits_successful_execution"
          },
          "last_execution" : {
            "timestamp" : "2020-04-24T00:40:43.516Z",
            "successful" : true
          },
          "last_successful_execution" : {
            "timestamp" : "2020-04-24T00:40:43.516Z",
            "successful" : true
          },
          "last_throttle" : {
            "reason" : "throttling interval is set to [10m] but time elapsed since last execution is [59.6s]",
            "timestamp" : "2020-04-24T00:41:43.159Z"
          }
        }
      },
      "headers" : {
        "_xpack_security_authentication" : "49itAwALemh1LnhpYW9sZWkGCndyaXRlcl9kZXYNa2liYW5hX3N5c3RlbQ9tb25pdG9yaW5nX3VzZXIWbWFjaGluZV9sZWFybmluZ19hZG1pbg13YXRjaGVyX2FkbWluDmNsdXN0ZXJfbWFuYWdlCgABCnpodXhpYW9sZWkBAAEAJWVzLWNuLXYwaDFlMmx2MDAwMnJjZ2VoLTI4M2JjN2U2LTAwMDIHbmF0aXZlMQZuYXRpdmUAAAoA"
      },
      "version" : -1,
      "last_checked" : "2020-04-24T00:57:43.283Z",
      "execution_state" : "execution_not_needed",
      "last_met_condition" : "2020-04-24T00:41:43.159Z"
    }
  }
}

• 常见返回错误
  

• 修正数据

POST event_user/_update_by_query?pipeline=cid_20181
{
  "query": {
    "bool": {
      "must": [
        {
          "range": {
            "datatime": {
              "gte": "2020-08-18",
              "time_zone": "+08:00",
              "format": "yyyy-MM-dd"
            }
          }
        },
        {
          "terms": {
            "token.number": [
              "198513907208"
            ]
          }
        },
        {
          "term": {
            "token.pd_created": {
              "value": true
            }
          }
        },
        {
          "term": {
            "action": {
              "value": "user_login"
            }
          }
        }
      ]
    }
  }
}


PUT _ingest/pipeline/cid_20181
{
  "processors": [
    {
      "script": {
        "source": """
        ctx.token.cid=20181;
        ctx.token.channel_title="fq-duanxinlahuo01_cpc_sj_20181";
        ctx.token.channel_abbr="乐享借短信拉活";
        
        """
      }
    }
  ]
}