k8s filebeat收集容器中的日志文件

k8s-filebeat收集容器中的日志文件

1. K8s-收集容器中的日志文件

• 收集容器文件图解
  

  针对容器中日志文件：在Pod中增加一个容器运行日志采集器，使用emtyDir共享日志目录让日志采集器读取到日志文件

• 示例代码

  [root@k8s-master elk]# vi app-log-logfile.yaml
  [root@k8s-master elk]# cat app-log-logfile.yaml 
  apiVersion: apps/v1
  kind: Deployment
  metadata:
    name: app-log-logfile
  spec:
    replicas: 3
    selector:
      matchLabels:
        project: microservice
        app: nginx-logfile
    template:
      metadata:
        labels:
          project: microservice
          app: nginx-logfile
      spec:
        containers:
        # 应用容器
        - name: nginx 
          image: lizhenliang/nginx-php
          # 将数据卷挂载到日志目录
          volumeMounts:
          - name: nginx-logs 
            mountPath: /usr/local/nginx/logs
        # 日志采集器容器
        - name: filebeat
          image: elastic/filebeat:7.10.1 
          args: [
            "-c", "/etc/filebeat.yml",
            "-e",
          ]
          resources:
            requests:
              cpu: 100m
              memory: 100Mi
            limits:
              memory: 500Mi
          securityContext:
            runAsUser: 0
          volumeMounts:
          # 挂载filebeat配置文件
          - name: filebeat-config
            mountPath: /etc/filebeat.yml
            subPath: filebeat.yml
          # 将数据卷挂载到日志目录
          - name: nginx-logs 
            mountPath: /usr/local/nginx/logs
        # 数据卷共享日志目录
        volumes:
        - name: nginx-logs
          emptyDir: {}
        - name: filebeat-config
          configMap:
            name: filebeat-nginx-config
  ---
  apiVersion: v1
  kind: Service
  metadata:
    name: app-log-logfile
  spec:
    ports:
    - port: 80
      protocol: TCP
      targetPort: 80
    selector:
      project: microservice
      app: nginx-logfile
  ---
  apiVersion: v1
  kind: ConfigMap
  metadata:
    name: filebeat-nginx-config
    
  data:
    # 配置文件保存在ConfigMap
    filebeat.yml: |-
      filebeat.inputs:
        - type: log
          paths:
            - /usr/local/nginx/logs/access.log
          # tags: ["access"]
          fields_under_root: true
          fields:
            project: microservice
            app: nginx
  
      setup.ilm.enabled: false
      setup.template.name: "nginx-access"
      setup.template.pattern: "nginx-access-*"
  
      output.elasticsearch:
        hosts: ['127.0.0.1:9200']
        username: "admin"
        password: "12345678"
        index: "nginx-access-%{+yyyy.MM.dd}"
  

2. 案例

• 编写配置文件

  [root@k8s-master elk]# vi app-log-logfile.yaml
  [root@k8s-master elk]# cat app-log-logfile.yaml 
  apiVersion: apps/v1
  kind: Deployment
  metadata:
    name: app-log-logfile
  spec:
    replicas: 3
    selector:
      matchLabels:
        project: microservice
        app: nginx-logfile
    template:
      metadata:
        labels:
          project: microservice
          app: nginx-logfile
      spec:
        containers:
        # 应用容器
        - name: nginx 
          image: lizhenliang/nginx-php
          # 将数据卷挂载到日志目录
          volumeMounts:
          - name: nginx-logs 
            mountPath: /usr/local/nginx/logs
        # 日志采集器容器
        - name: filebeat
          image: elastic/filebeat:7.10.1 
          args: [
            "-c", "/etc/filebeat.yml",
            "-e",
          ]
          resources:
            requests:
              cpu: 100m
              memory: 100Mi
            limits:
              memory: 500Mi
          securityContext:
            runAsUser: 0
          volumeMounts:
          # 挂载filebeat配置文件
          - name: filebeat-config
            mountPath: /etc/filebeat.yml
            subPath: filebeat.yml
          # 将数据卷挂载到日志目录
          - name: nginx-logs 
            mountPath: /usr/local/nginx/logs
        # 数据卷共享日志目录
        volumes:
        - name: nginx-logs
          emptyDir: {}
        - name: filebeat-config
          configMap:
            name: filebeat-nginx-config
  ---
  apiVersion: v1
  kind: Service
  metadata:
    name: app-log-logfile
  spec:
    ports:
    - port: 80
      protocol: TCP
      targetPort: 80
    selector:
      project: microservice
      app: nginx-logfile
  ---
  apiVersion: v1
  kind: ConfigMap
  metadata:
    name: filebeat-nginx-config
    
  data:
    # 配置文件保存在ConfigMap
    filebeat.yml: |-
      filebeat.inputs:
        - type: log
          paths:
            - /usr/local/nginx/logs/access.log
          # tags: ["access"]
          fields_under_root: true
          fields:
            project: microservice
            app: nginx
  
      setup.ilm.enabled: false
      setup.template.name: "nginx-access"
      setup.template.pattern: "nginx-access-*"
  
      output.elasticsearch:
        hosts: ['127.0.0.1:9200']
        username: "admin"
        password: "12345678"
        index: "nginx-access-%{+yyyy.MM.dd}"
  

• 运行配置文件

  [root@k8s-master elk]# kubectl apply -f app-log-logfile.yaml 
  deployment.apps/app-log-logfile created
  service/app-log-logfile created
  configmap/filebeat-nginx-config created
  

• 查看容器是否启动

  [root@k8s-master elk]# kubectl get pods
  NAME                               READY   STATUS    RESTARTS   AGE
  app-log-logfile-68b99bf447-fvvdj   2/2     Running   0          2m21s
  

• 访问nginx

  [root@k8s-master elk]# kubectl get pods -o wide
  NAME                               READY   STATUS    RESTARTS   AGE     IP              NODE         NOMINATED NODE   READINESS GATES
  app-log-logfile-68b99bf447-fvvdj   2/2     Running   0          5m53s   10.244.85.199   k8s-node01   <none>           <none>
  nginx                              1/1     Running   0          33h     10.244.85.196   k8s-node01   <none>           <none>
  tomcat                             1/1     Running   0          33h     10.244.85.197   k8s-node01   <none>           <none>
  web-5df8b97c79-hksfc               1/1     Running   0          3d4h    10.244.85.195   k8s-node01   <none>           <none>
  [root@k8s-master elk]# curl -I 10.244.85.199/status.html
  HTTP/1.1 200 OK
  Server: nginx
  Date: Thu, 08 Jul 2021 14:41:48 GMT
  Content-Type: text/html
  Content-Length: 3
  Last-Modified: Tue, 01 Jan 2019 20:08:29 GMT
  Connection: close
  ETag: "5c2bc8bd-3"
  Accept-Ranges: bytes
  [root@k8s-master elk]# curl -I 10.244.85.199/status.html
  HTTP/1.1 200 OK
  Server: nginx
  Date: Thu, 08 Jul 2021 14:41:48 GMT
  Content-Type: text/html
  Content-Length: 3
  Last-Modified: Tue, 01 Jan 2019 20:08:29 GMT
  Connection: close
  ETag: "5c2bc8bd-3"
  Accept-Ranges: bytes
  

• kibana验证是否有索引
  

• 创建索引验证数据