ELK 收集 K8S (containerd 容器运行时) 四

优化 filebeat 采集的日志

现实情况下，filebeat 采集过多无用的日志会造成 CPU，内存，带宽的浪费，尽量控制采集有用的日志

根据实际业务情况，这边控制采集 K8S 4个命名空间下的日志，其他命名空间的日志抛弃

cat cm.yaml
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: filebeat-config
  namespace: kube-system
  labels:
    k8s-app: filebeat
data:
  filebeat.yml: |-
    setup.ilm.enabled: false
    filebeat.inputs:
    - type: container
      paths:
        - /var/log/containers/*.log
      processors:
        - add_kubernetes_metadata:
            # 添加k8s描述字段
            default_indexers.enabled: true
            default_matchers.enabled: true
            host: ${NODE_NAME}
            matchers:
            - logs_path:
                logs_path: "/var/log/containers/"
        - drop_fields:
            # 删除的多余字段
            fields: ["host", "tags", "ecs", "log", "prospector", "agent", "input", "beat", "offset"]
            ignore_missing: true

      multiline.pattern: '^[[:space:]]+(at|\.{3})\b|^Caused by:'
      multiline.negate: false
      multiline.match: after


    setup.template.name: "k8s"
    setup.template.pattern: "k8s-*"
    setup.template.enabled: false
    # 如果是第一次则不需要, 如果 index-template 已经存在需要更新, 则需要
    setup.template.overwrite: false
    setup.template.settings:
      # 根据收集的日志量级, 因为日志会每天一份, 如果一天的日志量小于 30g, 一个 shard 足够
      index.number_of_shards: 2
      # 这个日志并不是那么重要, 并且如果是单节点的话, 直接设置为 0 个副本
      index.number_of_replicas: 0

    output.kafka:
      hosts: ['kafka-svc:9092']
      # 启动进程数
      worker: 20
      # 发送重试的次数取决于max_retries的设置默认为3
      max_retries: 3
      # 单个elasticsearch批量API索引请求的最大事件数。默认是50。
      bulk_max_size: 800
      topics:
        - topic: "k8s-%{[kubernetes.namespace]}-%{[kubernetes.container.name]}-%{+yyyy.MM.dd}"
          when.equals:
            kubernetes.namespace: "openfaas-reform-fn"
        - topic: "k8s-%{[kubernetes.namespace]}-%{[kubernetes.container.name]}-%{+yyyy.MM.dd}"
          when.equals:
            kubernetes.namespace: "pre-nengguan"
        - topic: "k8s-%{[kubernetes.namespace]}-%{[kubernetes.container.name]}-%{+yyyy.MM.dd}"
          when.equals:
            kubernetes.namespace: "shenshou"
        - topic: "k8s-%{[kubernetes.namespace]}-%{[kubernetes.container.name]}-%{+yyyy.MM.dd}"
          when.equals:
            kubernetes.namespace: "test-nengguan"


    setup.kibana:
      host: ':'

    # 设置 ilm 的 policy life, 日志保留
    setup.ilm:
      policy_file: /etc/indice-lifecycle.json

---
apiVersion: v1
kind: ConfigMap
metadata:
  name: filebeat-index-rules
  namespace: kube-system
  labels:
    k8s-app: filebeat
data:
  indice-lifecycle.json: |-
    {
      "policy": {
        "phases": {
          "hot": {
            "actions": {
              "rollover": {
                "max_size": "5GB" ,
                "max_age": "1d"
              }
            }
          },
          "delete": {
            "min_age": "5d",
            "actions": {
              "delete": {}
            }
          }
        }
      }
    }

可参考：
https://www.elastic.co/guide/en/beats/filebeat/current/kafka-output.html
https://www.elastic.co/guide/en/beats/filebeat/current/exported-fields-kubernetes-processor.html
https://www.elastic.co/guide/en/beats/filebeat/current/exported-fields.html
https://www.elastic.co/guide/en/beats/filebeat/current/defining-processors.html#conditions