通过helm部署EFK收集应用日志，ingressnginx日志解析。

前段时间看了马哥的k8s新书，最后几章讲了下EFK，尝试部署了下，很多问题， 这里改进下，写个笔记记录下吧。

准备工作

所有组件都通过helm3部署，先添加几个仓库。

helm repo add bitnami https://charts.bitnami.com/bitnami
helm repo add fluent https://fluent.github.io/helm-charts

存储用的nfs的sc，自行解决。

三个4G内存工作节点的k8s集群。

部署es

其中动态pv, prometheus 根据实际情况修改，没装prometheus也没什么事。

 

[root@bjzb-lecar-ops-jenkins-master-33 cluster-log]# cat bitnami-elasticsearch-values.yaml
clusterDomain: cluster.local   # Kubernetes集群域名；
name: elasticsearch   # Elasticsearch集群名称；

master:   # 准主节点相关的配置；
  name: master
  replicas: 2   # 实例数量；
  heapSize: 512m   # 堆内存大小；
  resources:
    limits: {}
    #   cpu: 1000m
    #   memory: 2048Mi
    requests:
      cpu: 200m
      memory: 512Mi
  persistence:   # 持久卷相关的配置；
    enabled: true    # 禁用时将自动使用emptyDir存储卷；
    storageClass: "managed-nfs-storage"   # 从指定存储类中动态创建PV；
    # existingClaim: my-persistent-volume-claim   # 使用现有的PVC；
    # existingVolume: my-persistent-volume    # 使用现有的PV；
    accessModes:
      - ReadWriteOnce
    size: 8Gi
  service:   # 服务配置
    type: ClusterIP
    port: 9300   # 节点间的transport流量使用端口；

coordinating:   # 仅协调节点相关的配置；
  replicas: 2   # 实例数量；
  heapSize: 128m
  resources:
    requests:
      cpu: 250m
      memory: 512Mi
  service:   # 仅协调节点相关的服务，这也是接收Elasticsearch客户端请求的入口；
    type: ClusterIP
    port: 9200
    # nodePort:
    # loadBalancerIP:

data:   # 数据节点相关的配置；
  name: data
  replicas: 2
  heapSize: 512m 
  resources:   # 数据节点是CPU密集及IO密集型的应用，资源需求和限制要谨慎设定；
    limits: {}
    #   cpu: 100m
    #   memory: 2176Mi
    requests:
      cpu: 250m
      memory: 512Mi
  persistence:
    enabled: true
    storageClass: "managed-nfs-storage"
    # existingClaim: my-persistent-volume-claim
    # existingVolume: my-persistent-volume
    accessModes:
      - ReadWriteOnce
    size: 10Gi

ingest:   # 摄取节点相关的配置；
  enabled: false   # 默认为禁用状态；
  name: ingest
  replicas: 2
  heapSize: 128m
  resources:
    limits: {}
    #   cpu: 100m
    #   memory: 384Mi
    requests:
      cpu: 500m
      memory: 512Mi
  service:
    type: ClusterIP
    port: 9300

curator:   # curator相关的配置；
  enabled: false
  name: curator
  cronjob:   # 执行周期及相关的配置；
    # At 01:00 every day
    schedule: "0 1 * * *"
    concurrencyPolicy: ""
    failedJobsHistoryLimit: ""
    successfulJobsHistoryLimit: ""
    jobRestartPolicy: Never

metrics:   # 用于暴露指标的exporter；
  enabled: true
  name: metrics
  service:
    type: ClusterIP
    annotations:   # 指标采集相关的注解信息；
      prometheus.io/scrape: "true"
      prometheus.io/port: "9114"
  resources:
    limits: {}
    #   cpu: 100m
    #   memory: 128Mi
    requests:
       cpu: 100m
       memory: 128Mi
  podAnnotations:   # Pod上的注解，用于支持指标采集；
    prometheus.io/scrape: "true"
    prometheus.io/port: "8080"
  serviceMonitor:   # Service监控相关的配置
    enabled: false
    namespace: monitoring
    interval: 10s
    scrapeTimeout: 10s

helm install es -f bitnami-elasticsearch-values.yaml  bitnami/elasticsearch -n logging

哎，这一步各种问题，会遇到镜像下载慢，k8s集群资源不够（我已经把yml里申请的资源调的很低了），存储权限问题，反正大家注意点就行。

部署fluent-bit

[root@bj-k8s-master efk]# cat fluent-fluent-bit-values.yaml 
# kind -- DaemonSet or Deployment
kind: DaemonSet

image:
  repository: fluent/fluent-bit
  pullPolicy: IfNotPresent

service:
  type: ClusterIP
  port: 2020
  annotations:
    prometheus.io/path: "/api/v1/metrics/prometheus"
    prometheus.io/port: "2020"
    prometheus.io/scrape: "true"

resources: {}
  # limits:
  #   cpu: 100m
  #   memory: 128Mi
  #requests:
  #  cpu: 100m
  #  memory: 128Mi

tolerations:
  - key: node-role.kubernetes.io/master
    effect: NoSchedule

config:
  service: |
    [SERVICE]
        Flush 3
        Daemon Off
        Log_Level info
        #Log_Level debug
        Parsers_File custom_parsers.conf
        Parsers_File parsers.conf
        HTTP_Server On
        HTTP_Listen 0.0.0.0
        HTTP_Port 2020

  inputs: |
    [INPUT]
        Name tail
        Path /var/log/containers/*.log
        Parser docker
        Tag kube.*
        Mem_Buf_Limit 5MB
        Skip_Long_Lines On
        Refresh_Interval  10
    [INPUT]
        Name tail
        Path /var/log/containers/nginx-demo*.log
        Parser docker
        Tag nginx-demo.*
        Mem_Buf_Limit 5MB
        Skip_Long_Lines On
        Refresh_Interval  10
    [INPUT]
        Name tail
        Path /var/log/containers/ingress-nginx-controller*.log
        Parser docker
        Tag ingress-nginx-controller.*
        Mem_Buf_Limit 5MB
        Skip_Long_Lines On
        Refresh_Interval  10

  filters: |
    [FILTER]
        Name                kubernetes
        Match               kube.*
        Kube_URL            https://kubernetes.default.svc:443
        Kube_CA_File        /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        Kube_Token_File     /var/run/secrets/kubernetes.io/serviceaccount/token
        Kube_Tag_Prefix     kube.var.log.containers.
        Merge_Log           On
        Keep_Log            Off
        K8S-Logging.Exclude On
        K8S-Logging.Parser On
    [FILTER]
        Name                kubernetes
        Match               ingress-nginx-controller.*
        Kube_URL            https://kubernetes.default.svc:443
        Kube_CA_File        /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        Kube_Token_File     /var/run/secrets/kubernetes.io/serviceaccount/token
        Kube_Tag_Prefix     kube.var.log.containers.
        Merge_Log           On
        Merge_Parser        ingress-nginx
        Keep_Log            Off
        K8S-Logging.Exclude On
        K8S-Logging.Parser On


  outputs: |
    [OUTPUT]
        Name es
        Match kube.*
        Host es-elasticsearch-coordinating-only.logging.svc.cluster.local.
        Logstash_Format On
        Logstash_Prefix k8s-cluster
        Type  flb_type
        Replace_Dots On

    [OUTPUT]
        Name es
        Match nginx-demo.*
        Host es-elasticsearch-coordinating-only.logging.svc.cluster.local.
        Logstash_Format On
        Logstash_Prefix nginx-demo
        Type  flb_type
        Replace_Dots On
    [OUTPUT]
        Name es
        Match ingress-nginx-controller.*
        Host es-elasticsearch-coordinating-only.logging.svc.cluster.local.
        Logstash_Format On
        Logstash_Prefix ingress-nginx-controller
        Type  flb_type
        Replace_Dots On


  customParsers: |
    [PARSER]
        Name docker_no_time
        Format json
        Time_Keep Off
        Time_Key time
        Time_Format %Y-%m-%dT%H:%M:%S.%L

    [PARSER]
        Name        ingress-nginx
        Format      regex
        Regex       ^(?<message>(?<remote>[^ ]*) - (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*) "(?<referer>[^\"]*)" "(?<agent>[^\"]*)" (?<request_length>[^ ]*) (?<request_time>[^ ]*) \[(?<proxy_upstream_name>[^ ]*)\] \[(?<proxy_alternative_upstream_name>[^ ]*)\] (?<upstream_addr>[^ ]*) (?<upstream_response_length>[^ ]*) (?<upstream_response_time>[^ ]*) (?<upstream_status>[^ ]*) (?<req_id>[^ ]*).*)$
        Time_Key    time
        Time_Format %d/%b/%Y:%H:%M:%S %z

helm install fb -f fluent-fluent-bit-values.yaml fluent/fluent-bit -n logging

其中nginx-demo是我自己部署的一个应用，拿来测试用的。注意下es的host地址，如果跟我不是一样的namespace记得改下，吃过亏。这个书里提供的都写一个index里，nginx-ingress日志也不解析，fluent-bit研究了半天，大概就这程度吧，通过docker不同的log名字打上tag，这样每个应用在es里单独的index。当然量不大都写进一个index，通过label字段查询某应用的日志也行。

部署kibana

[root@bj-k8s-master efk]# cat bitnami-kibana-values.yaml 
replicaCount: 1

updateStrategy:
  type: RollingUpdate

plugins:
  - https://github.com/pjhampton/kibana-prometheus-exporter/releases/download/7.8.1/kibana-prometheus-exporter-7.8.1.zip

persistence:
  enabled: true
  storageClass: "managed-nfs-storage"
  # existingClaim: your-claim
  accessMode: ReadWriteOnce
  size: 10Gi

service:
  port: 5601
  type: ClusterIP
  # nodePort:
  externalTrafficPolicy: Cluster
  annotations: {}
  # loadBalancerIP:
  # extraPorts:

ingress:
  enabled: true
  certManager: false
  annotations:
    kubernetes.io/ingress.class: nginx
  hostname: kibana.ilinux.io
  path: /
  tls: false
      # tlsHosts:
      #   - www.kibana.local
      #   - kibana.local
      # tlsSecret: kibana.local-tls

configuration:
  server:
    basePath: ""
    rewriteBasePath: false

metrics:
  enabled: true
  service:
    annotations:
      prometheus.io/scrape: "true"
      prometheus.io/port: "80"
      prometheus.io/path: "_prometheus/metrics"

  serviceMonitor:
    enabled: false
    # namespace: monitoring
    # interval: 10s
    # scrapeTimeout: 10s
    # selector:
    #   prometheus: my-prometheus

elasticsearch:
  hosts:
  - es-elasticsearch-coordinating-only.logging.svc.cluster.local.
  # - elasticsearch-2
  port: 9200

helm install kib -f bitnami-kibana-values.yaml bitnami/kibana -n logging

同上如果跟我namespace不一样记得改下es地址。 这地方书里有坑，把charts pull下来看了下，values.yml文件有出入，估计是写书的时候chart版本不一样导致的。

配置kibana

自己改下hosts解析到ingress地址，访问kibana

添加匹配的索引

每个应用日志独立的index

 

ingress-nginx日志已解析

fluentbit官方文档

https://docs.fluentbit.io/

在线正则匹配

https://rubular.com/