使用filebeat 收集日志到logstash 收集日志redis再到logstash到es

大型场合的工作流程图

filebeat -->logstash ---> redis ---> logstash --->es

工作环境：
需要两台logstash，

安装jdk8

[root@es-web1]# apt install openjdk-8-jdk -y

这里已经安装filebeat

配置filebeat（这里的输出只能写一个，如果之前已经存在有，需要注释，或者删除即可）

[root@es-web1]# vim /etc/filebeat/filebeat.yml

filebeat.inputs:
- type: log
  enabled: True
  paths:
    - /apps/nginx/logs/error.log
  fields:
    app: nginx-errorlog
    group: n223

- type: log
  enabled: True
  paths:
    - /var/log/nginx/access.log
  fields:
    app: nginx-accesslog
    group: n125

output.logstash:
  hosts: ["172.31.2.107:5044"]
  enabled: true
  worker: 1
  compression_level: 3
  loadbalance: true

重启

root@long:~# systemctl restart filebeat

上传deb包，安装

[root@es-web1 src]# dpkg -i logstash-7.12.1-amd64.deb

配置logstash1

[root@es-web1]# vim /etc/logstash/conf.d/beats.conf

input {
  beats {
    port => 5044
    codec => "json"
  }
}

output {
  if [fields][app] == "nginx-accesslog" {
    redis {
      data_type => "list"
      key => "long-n178-nginx-accesslog"
      host => "172.31.2.106"
      port => "6379"
      db => "3"
      password => "123456"
  }}

  if [fields][app] == "nginx-errorlog" {
    redis {
      data_type => "list"
      key => "long-n178-nginx-errorlog"                      
      host => "172.31.2.106"
      port => "6379"
      db => "3"
      password => "123456"
   }}
}

重启

[root@es-web1]# systemctl restart logstash

检查redis是否获取到信息

[root@es-redis]# redis-cli -h 172.31.2.106
172.31.2.106:6379> auth 123456
172.31.2.106:6379[3]> select 3
172.31.2.106:6379[3]> keys *
(empty list or set)

172.31.2.106:6379[3]> keys *
1) "long-n178-nginx-accesslog"
2) "long-n178-nginx-errorlog"

172.31.2.106:6379[3]> LPOP 

配置logstash2

[root@logstash2 ~]# vim /etc/logstash/conf.d/logstash-to-es.conf

input {
  redis {
    data_type => "list"
    key => "long-n178-nginx-accesslog"
    host => "172.31.2.106"
    port => "6379"
    db => "3"
    password => "123456"
  }

  redis {
    data_type => "list"
    key => "long-n178-nginx-errorlog"
    host => "172.31.2.106"
    port => "6379"
    db => "3"
    password => "123456"
  }
}

output {
  if [fields][app] == "nginx-accesslog" {
    elasticsearch {
      hosts => ["172.31.2.101:9200"]
      index => "long-logstash-nginx-accesslog-%{+YYYY.MM.dd}"
  }}

  if [fields][app] == "nginx-errorlog" {
    elasticsearch {
      hosts => ["172.31.2.101:9200"]
      index => "long-logstash-nginx-errorlog-%{+YYYY.MM.dd}" 
  }}
}

重启

[root@logstash2 ~]# systemctl restart logstash

添加到kibana

略