EFK 收集 Docker 日志

过程： filebeat(收集) -> elasticsearch(存储) -> kibana(展示)
优点：简单，快速，容易上手
缺点：filebeat 把收集到的日志全部存入 elasticsearch，日志量大，有并发问题

# 建立目录
mkdir -p /data/docker-compose/efk/ && cd /data/docker-compose/efk/
mkdir elasticsearch  filebeat  kibana

# docker-compose.yml 配置
cat docker-compose.yml 
version: '3.2'

services:
  elasticsearch:
    build:
      context: elasticsearch/
      args:
        ELK_VERSION: $ELK_VERSION
    ports:
      - "9200:9200"
      - "9300:9300"
    environment:
      ES_JAVA_OPTS: "-Xmx2048m -Xms2048m"
      ELASTIC_PASSWORD: elastic
      # Use single node discovery in order to disable production mode and avoid bootstrap checks
      # see https://www.elastic.co/guide/en/elasticsearch/reference/current/bootstrap-checks.html
      discovery.type: single-node
    volumes:
        - ${GLOBAL_APP_PATH}elasticsearch/data:/usr/share/elasticsearch/data
    networks:
      - elk

  kibana:
    build:
      context: kibana/
      args:
        ELK_VERSION: $ELK_VERSION
    ports:
      - "5601:5601"
    networks:
      - elk
    depends_on:
      - elasticsearch

  filebeat:
    build:
      context: filebeat/
    networks:
      - elk
    user: root
    volumes:
        - ${GLOBAL_APP_PATH}filebeat/config/filebeat.yml:/usr/share/filebeat/filebeat.yml:ro
        - /var/lib/docker/containers:/var/lib/docker/containers:ro
        - /var/run/docker.sock:/var/run/docker.sock:ro
    privileged: true

networks:
  elk:
    driver: bridge

volumes:
  elasticsearch:

# 环境变量配置
cat .env 
ELK_VERSION=7.3.1
GLOBAL_APP_PATH=/data/docker-compose/efk/

# 配置 elasticsearch
cd /data/docker-compose/efk/elasticsearch

cat Dockerfile 
ARG ELK_VERSION

# https://www.docker.elastic.co/
FROM docker.elastic.co/elasticsearch/elasticsearch:${ELK_VERSION}

# Add your elasticsearch plugins setup here
# Example: RUN elasticsearch-plugin install analysis-icu

mkdir data 
chown 1000.1000 data

# 配置 filebeat

cd /data/docker-compose/efk/filebeat

cat Dockerfile 
ARG ELK_VERSION
FROM docker.elastic.co/beats/filebeat:${ELK_VERSION}

mkdir config
cd config

cat filebeat.yml 
setup.ilm.enabled: false
filebeat.inputs:
- type: docker
  containers.ids:
    - "*"
  containers.paths:
    - "/var/lib/docker/containers/${data.docker.container.id}/*.log"
  multiline.pattern: '^[[:space:]]+(at|\.{3})\b|^Caused by:'
  multiline.negate: false
  multiline.match: after


processors:
  - add_docker_metadata:
      host: "unix:///var/run/docker.sock"

setup.template.name: "docker"
setup.template.pattern: "docker-*"
setup.template.enabled: false
# 如果是第一次则不需要, 如果 index-template 已经存在需要更新, 则需要
setup.template.overwrite: false
setup.template.settings:
  index.number_of_shards: 2
  index.number_of_replicas: 0
output.elasticsearch:
  hosts: ["elasticsearch:9200"]
  worker: 12
  # 单个elasticsearch批量API索引请求的最大事件数。默认是50。
  bulk_max_size: 400
  indices:
    - index: "docker-%{[container.name]}-%{+yyyy.MM.dd}"


# 配置 kibana
cd /data/docker-compose/efk/kibana

cat Dockerfile 
ARG ELK_VERSION

# https://www.docker.elastic.co/
FROM docker.elastic.co/kibana/kibana:${ELK_VERSION}

# Add your kibana plugins setup here
# Example: RUN kibana-plugin install <name|url>

启动

cd /data/docker-compose/efk
docker-compose build
docker-compose up -d

测试