过程: filebeat(收集) -> elasticsearch(存储) -> kibana(展示)
优点:简单,快速,容易上手
缺点:filebeat 把收集到的日志全部存入 elasticsearch,日志量大,有并发问题
# 建立目录
mkdir -p /data/docker-compose/efk/ && cd /data/docker-compose/efk/
mkdir elasticsearch filebeat kibana
# docker-compose.yml 配置
cat docker-compose.yml
version: '3.2'
services:
elasticsearch:
build:
context: elasticsearch/
args:
ELK_VERSION: $ELK_VERSION
ports:
- "9200:9200"
- "9300:9300"
environment:
ES_JAVA_OPTS: "-Xmx2048m -Xms2048m"
ELASTIC_PASSWORD: elastic
# Use single node discovery in order to disable production mode and avoid bootstrap checks
# see https://www.elastic.co/guide/en/elasticsearch/reference/current/bootstrap-checks.html
discovery.type: single-node
volumes:
- ${GLOBAL_APP_PATH}elasticsearch/data:/usr/share/elasticsearch/data
networks:
- elk
kibana:
build:
context: kibana/
args:
ELK_VERSION: $ELK_VERSION
ports:
- "5601:5601"
networks:
- elk
depends_on:
- elasticsearch
filebeat:
build:
context: filebeat/
networks:
- elk
user: root
volumes:
- ${GLOBAL_APP_PATH}filebeat/config/filebeat.yml:/usr/share/filebeat/filebeat.yml:ro
- /var/lib/docker/containers:/var/lib/docker/containers:ro
- /var/run/docker.sock:/var/run/docker.sock:ro
privileged: true
networks:
elk:
driver: bridge
volumes:
elasticsearch:
# 环境变量配置
cat .env
ELK_VERSION=7.3.1
GLOBAL_APP_PATH=/data/docker-compose/efk/
# 配置 elasticsearch
cd /data/docker-compose/efk/elasticsearch
cat Dockerfile
ARG ELK_VERSION
# https://www.docker.elastic.co/
FROM docker.elastic.co/elasticsearch/elasticsearch:${ELK_VERSION}
# Add your elasticsearch plugins setup here
# Example: RUN elasticsearch-plugin install analysis-icu
mkdir data
chown 1000.1000 data
# 配置 filebeat
cd /data/docker-compose/efk/filebeat
cat Dockerfile
ARG ELK_VERSION
FROM docker.elastic.co/beats/filebeat:${ELK_VERSION}
mkdir config
cd config
cat filebeat.yml
setup.ilm.enabled: false
filebeat.inputs:
- type: docker
containers.ids:
- "*"
containers.paths:
- "/var/lib/docker/containers/${data.docker.container.id}/*.log"
multiline.pattern: '^[[:space:]]+(at|\.{3})\b|^Caused by:'
multiline.negate: false
multiline.match: after
processors:
- add_docker_metadata:
host: "unix:///var/run/docker.sock"
setup.template.name: "docker"
setup.template.pattern: "docker-*"
setup.template.enabled: false
# 如果是第一次则不需要, 如果 index-template 已经存在需要更新, 则需要
setup.template.overwrite: false
setup.template.settings:
index.number_of_shards: 2
index.number_of_replicas: 0
output.elasticsearch:
hosts: ["elasticsearch:9200"]
worker: 12
# 单个elasticsearch批量API索引请求的最大事件数。默认是50。
bulk_max_size: 400
indices:
- index: "docker-%{[container.name]}-%{+yyyy.MM.dd}"
# 配置 kibana
cd /data/docker-compose/efk/kibana
cat Dockerfile
ARG ELK_VERSION
# https://www.docker.elastic.co/
FROM docker.elastic.co/kibana/kibana:${ELK_VERSION}
# Add your kibana plugins setup here
# Example: RUN kibana-plugin install <name|url>
启动
cd /data/docker-compose/efk
docker-compose build
docker-compose up -d
测试