ELK--02 使用模块收集日志
目录
1.收集多台服务器nginx日志
1.在别的服务器上面安装nginx
#更换官方源
[root@db02 ~]# cat /etc/yum.repos.d/nginx.repo
[nginx-stable]
name=nginx stable repo
baseurl=http://nginx.org/packages/centos/$releasever/$basearch/
gpgcheck=1
enabled=1
gpgkey=https://nginx.org/keys/nginx_signing.key
module_hotfixes=true
[nginx-mainline]
name=nginx mainline repo
baseurl=http://nginx.org/packages/mainline/centos/$releasever/$basearch/
gpgcheck=1
enabled=0
gpgkey=https://nginx.org/keys/nginx_signing.key
module_hotfixes=true
#安装nginx
[root@db02 ~]# yum install nginx -y
2.复制db01的nginx的配置文件
[root@db02 ~]# scp 10.0.0.51:/etc/nginx/nginx.conf /etc/nginx/nginx.conf
[root@db02 ~]# scp 10.0.0.51:/etc/nginx/conf.d/www.conf /etc/nginx/conf.d/
3.创建测试页面
[root@db02 ~]# mkdir /code/www/ -p
[root@db02 ~]# echo "db02-www" > /code/www/index.html
4.重启nginx
[root@db02 ~]# >/var/log/nginx/access.log
[root@db02 ~]# >/var/log/nginx/error.log
[root@db02 ~]# nginx -t
[root@db02 ~]# systemctl restart nginx
5.安装filebeat
[root@db02 ~]# rpm -ivh filebeat-6.6.0-x86_64.rpm
6.复制filebeat配置文件
[root@db02 ~]# scp 10.0.0.51:/etc/filebeat/filebeat.yml /etc/filebeat/
7.启动filebeat
[root@db02 ~]# systemctl restart filebeat
8.生成测试数据
[root@db02 ~]# curl 127.0.0.1/22222222222222
[root@db02 ~]# curl 127.0.0.1
#收集nginx完整的filebeat配置
[root@db01]# cat /etc/filebeat/filebeat.yml
filebeat.inputs:
- type: log
enabled: true
paths:
- /var/log/nginx/access.log
json.keys_under_root: true
json.overwrite_keys: true
- type: log
enabled: true
paths:
- /var/log/nginx/error.log
output.elasticsearch:
hosts: ["10.0.0.51:9200"]
indices:
- index: "nginx-access-%{[beat.version]}-%{+yyyy.MM}"
when.contains:
source: "/var/log/nginx/access.log"
- index: "nginx-error-%{[beat.version]}-%{+yyyy.MM}"
when.contains:
source: "/var/log/nginx/error.log"
setup.template.name: "nginx"
setup.template.pattern: "nginx-*"
setup.template.enabled: false
setup.template.overwrite: true
2.filebeat收集tomcat的json日志
1.安装tomcat
[root@db01 ~]# yum install tomcat -y
[root@db01 ~]# systemctl start tomcat
[root@db01 ~]# tail -f /var/log/tomcat/localhost_access_log.2020-02-14.txt
2.修改tomcat配置将日志转换为json格式
[root@db01 ~]# cp /etc/tomcat/server.xml /opt/
[root@db01 ~]# vim /etc/tomcat/server.xml 139行
pattern="{"clientip":"%h","ClientUser":"%l","authenticated":"%u","AccessTime":"%t","method":"%r","status":"%s","SendBytes":"%b","Query?string":"%q","partner":"%{Referer}i","AgentVersion":"%{User-Agent}i"}"/>
3.清空日志并重启
[root@db01 ~]# > /var/log/tomcat/localhost_access_log.2020-02-14.txt
[root@db01 ~]# systemctl restart tomcat
4.访问并查看日志是否为json格式
[root@db01 ~]# curl 127.0.0.1:8080
[root@db01 ~]# tail -f /var/log/tomcat/localhost_access_log.2020-02-14.txt
5.创建filebeat配置文件 ======== （nginx+tomcat的filebeat配置文件）
[root@db01 ~]# cat >/etc/filebeat/filebeat.yml <<EOF
filebeat.inputs:
- type: log
enabled: true
paths:
- /var/log/nginx/access.log
json.keys_under_root: true
json.overwrite_keys: true
tags: ["access"]
- type: log
enabled: true
paths:
- /var/log/nginx/error.log
tags: ["error"]
- type: log
enabled: true
paths:
- /var/log/tomcat/localhost_access_log.*.txt
json.keys_under_root: true
json.overwrite_keys: true
tags: ["tomcat"]
output.elasticsearch:
hosts: ["10.0.0.51:9200"]
indices:
- index: "nginx-access-%{[beat.version]}-%{+yyyy.MM}"
when.contains:
tags: "access"
- index: "nginx-error-%{[beat.version]}-%{+yyyy.MM}"
when.contains:
tags: "error"
- index: "tomcat-%{[beat.version]}-%{+yyyy.MM}"
when.contains:
tags: "tomcat"
setup.template.name: "nginx"
setup.template.pattern: "nginx-*"
setup.template.enabled: false
setup.template.overwrite: true
EOF
6.重启filebeat并检查
[root@db01 ~]# systemctl restart filebeat
#filebeat收集tomcat配置文件
[root@db01 ~]# cat /etc/filebeat/filebeat.yml
filebeat.inputs:
- type: log
enabled: true
paths:
- /var/log/tomcat/localhost_access_log.*.txt
json.keys_under_root: true
json.overwrite_keys: true
tags: ["tomcat"]
output.elasticsearch:
hosts: ["10.0.0.51:9200"]
indices:
- index: "tomcat-%{[beat.version]}-%{+yyyy.MM}"
when.contains:
tags: "tomcat"
3.filebeat收集java多行匹配模式
#es手机java官方地址
https://www.elastic.co/guide/en/beats/filebeat/6.6/multiline-examples.html
1.filebeat配置文件
cat >/etc/filebeat/filebeat.yml<<EOF
filebeat.inputs:
- type: log
enabled: true
paths:
- /var/log/elasticsearch/elasticsearch.log
multiline.pattern: '^['
multiline.negate: true
multiline.match: after
output.elasticsearch:
hosts: ["10.0.0.51:9200"]
index: "es-%{[beat.version]}-%{+yyyy.MM}"
setup.template.name: "es"
setup.template.pattern: "es-*"
setup.template.enabled: false
setup.template.overwrite: true
EOF
2.重启filebeat
systemctl restart filebeat
3.制造报错日志
#更改es的配置文件并重启制造报错日志
4.检查java报错日志是否合并成一行了
kibana添加索引然后搜索关键词 at org
#filebeat收集java多长日志配置文件
[root@db01 ~]# cat /etc/filebeat/filebeat.yml
filebeat.inputs:
- type: log
enabled: true
paths:
- /var/log/elasticsearch/elasticsearch.log
multiline.pattern: '^['
multiline.negate: true
multiline.match: after
output.elasticsearch:
hosts: ["10.0.0.51:9200"]
index: "es-%{[beat.version]}-%{+yyyy.MM}"
setup.template.name: "es"
setup.template.pattern: "es-*"
setup.template.enabled: false
setup.template.overwrite: true
4.filbeat使用模块收集nginx日志
1.清空并把nginx日志恢复成普通格式
#清空日志
[root@db01 ~]# > /var/log/nginx/access.log
#编辑配置文件
[root@db01 ~]# vim /etc/nginx/nginx.conf
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
access_log /var/log/nginx/access.log main;
#检查并重启
[root@db01 ~]# nginx -t
[root@db01 ~]# systemctl restart nginx
2.访问并检查日志是否为普通格式
[root@db01 ~]# curl 127.0.0.1
[root@db01 ~]# tail -f /var/log/nginx/access.log
3.配置filebeat配置文件支持模块
[root@db01 ~]# cat /etc/filebeat/filebeat.yml
filebeat.config.modules:
path: ${path.config}/modules.d/*.yml
reload.enabled: true
reload.period: 10s
output.elasticsearch:
hosts: ["10.0.0.51:9200"]
indices:
- index: "nginx-access-%{[beat.version]}-%{+yyyy.MM}"
when.contains:
event.dataset: "nginx.access"
- index: "nginx-error-%{[beat.version]}-%{+yyyy.MM}"
when.contains:
event.dataset: "nginx.error"
setup.template.name: "nginx"
setup.template.pattern: "nginx-*"
setup.template.enabled: false
setup.template.overwrite: true
4.激活filebeat的nginx模块
[root@db01 ~]# filebeat modules enable nginx
[root@db01 ~]# filebeat modules list
[root@db01 ~]# ll /etc/filebeat/modules.d/nginx.yml
-rw-r--r-- 1 root root 369 Jan 24 2019 /etc/filebeat/modules.d/nginx.yml
5.配置filebeat的nginx模块配置文件
[root@db01 ~]# cat >/etc/filebeat/modules.d/nginx.yml <<EOF
- module: nginx
access:
enabled: true
var.paths: ["/var/log/nginx/access.log"]
error:
enabled: true
var.paths: ["/var/log/nginx/error.log"]
EOF
6.es安装filebeat的nginx模块必要插件并重启
#上传插件
[root@db01 ~]# ll
-rw-r--r-- 1 root root 33255554 Jan 8 08:15 ingest-geoip-6.6.0.zip
-rw-r--r-- 1 root root 62173 Jan 8 08:15 ingest-user-agent-6.6.0.zip
#切换目录并安装插件
[root@db01 ~]# cd /usr/share/elasticsearch/
[root@db01 ~]# ./bin/elasticsearch-plugin install file:///root/ingest-geoip-6.6.0.zip
注意安装时候需要输入 “y” 确认
[root@db01 ~]# ./bin/elasticsearch-plugin install file:///root/ingest-user-agent-6.6.0.zip
[root@db01 ~]# systemctl restart elasticsearch
7.重启filebeat
[root@db01 ~]# systemctl restart filebeat
8.删除es-head插件中原有nginx的数据和ibana中的ngixn数据
生成新的日志数据，es-head插件更新查看，kibana添加
5.filebeat使用模块收集mysql慢日志
#二进制安装
1.下载或上传软件包
wget https://downloads.mysql.com/archives/get/file/mysql-5.6.44-linux-glibc2.12-x86_64.tar.gz
2.解压
[root@db01 ~]# tar xf mysql-5.6.44-linux-glibc2.12-x86_64.tar.gz
[root@db01 ~]# ll
total 321404
drwxr-xr-x 13 root root 191 Oct 31 04:31 mysql-5.6.44-linux-glibc2.12-x86_64
-rw-r--r-- 1 root root 329105487 Oct 30 10:23 mysql-5.6.44-linux-glibc2.12-x86_64.tar.gz
3.安装依赖软件包
[root@db01 ~]# yum install -y autoconf libaio*
4.创建 mysql 用户
[root@db01 ~]# useradd mysql -s /sbin/nologin -M
[root@db01 ~]# id mysql
uid=1000(mysql) gid=1000(mysql) groups=1000(mysql)
5.将解压后的软件包目录移动到 /opt 目录下面并更改文件名
[root@db01 ~]# mv mysql-5.6.44-linux-glibc2.12-x86_64 /opt/mysql-5.6.44
[root@db01 ~]# cd /opt/mysql-5.6.44/
[root@db01 /opt/mysql-5.6.44]# ll
total 40
drwxr-xr-x 2 root root 4096 Oct 31 04:31 bin
-rw-r--r-- 1 7161 31415 17987 Mar 15 2019 COPYING
drwxr-xr-x 3 root root 18 Oct 31 04:30 data
drwxr-xr-x 2 root root 55 Oct 31 04:30 docs
drwxr-xr-x 3 root root 4096 Oct 31 04:30 include
drwxr-xr-x 3 root root 316 Oct 31 04:31 lib
drwxr-xr-x 4 root root 30 Oct 31 04:30 man
drwxr-xr-x 10 root root 291 Oct 31 04:30 mysql-test
-rw-r--r-- 1 7161 31415 2496 Mar 15 2019 README
drwxr-xr-x 2 root root 30 Oct 31 04:30 scripts
drwxr-xr-x 28 root root 4096 Oct 31 04:31 share
drwxr-xr-x 4 root root 4096 Oct 31 04:31 sql-bench
drwxr-xr-x 2 root root 136 Oct 31 04:30 support-files
6.制作软连接
[root@db01 ~]# ln -s /opt/mysql-5.6.44/ /opt/mysql
[root@db01 ~]# ll /opt/mysql
lrwxrwxrwx 1 root root 18 Oct 31 04:37 /opt/mysql -> /opt/mysql-5.6.44/
7.拷贝启动脚本
[root@db01 /opt/mysql-5.6.44]# cd /opt/mysql-5.6.44/support-files/
[root@db01 /opt/mysql-5.6.44/support-files]# cp mysql.server /etc/init.d/mysqld
[root@db01 /opt/mysql-5.6.44/support-files]# ll /etc/init.d/mysqld
-rwxr-xr-x 1 root root 10565 Oct 31 04:40 /etc/init.d/mysqld
8.拷贝配置文件
[root@db01 /opt/mysql-5.6.44/support-files]# cp my-default.cnf /etc/my.cnf
cp: overwrite ‘/etc/my.cnf’? y
[root@db01 /opt/mysql-5.6.44/support-files]# ll /etc/my.cnf
-rw-r--r--. 1 root root 1126 Oct 31 04:41 /etc/my.cnf
9.初始化数据库
[root@db01 /opt/mysql-5.6.44/support-files]# cd ../scripts/
[root@db01 /opt/mysql-5.6.44/scripts]# ll
total 36
-rwxr-xr-x 1 7161 31415 34558 Mar 15 2019 mysql_install_db
[root@db01 /opt/mysql-5.6.44/scripts]# ./mysql_install_db --basedir=/opt/mysql --datadir=/opt/mysql/data --user=mysql
#只要有两个ok就行
10.授权 mysql 目录
[root@db01 /opt/mysql-5.6.44/scripts]# chown -R mysql.mysql /opt/mysql-5.6.44/
[root@db01 /opt/mysql-5.6.44/scripts]# ll /opt/
total 0
lrwxrwxrwx 1 mysql mysql 18 Oct 31 04:37 mysql -> /opt/mysql-5.6.44/
drwxr-xr-x 13 mysql mysql 223 Oct 31 04:43 mysql-5.6.44
11.修改 mysql 启动脚本和程序
[root@db01 /opt/mysql-5.6.44/scripts]# sed -i 's#/usr/local#/opt#g' /etc/init.d/mysqld /opt/mysql/bin/mysqld_safe
12.启动 mysqkl
[root@db01 /opt/mysql-5.6.44/scripts]# /etc/init.d/mysqld start
Starting MySQL.Logging to '/opt/mysql/data/db01.err'.
SUCCESS!
13.添加环境变量
[root@db01 /opt/mysql-5.6.44/scripts]# vim /etc/profile.d/mysql.sh
export PATH="/opt/mysql/bin:$PATH"
[root@db01 /opt/mysql-5.6.44/scripts]# source /etc/profile.d/mysql.sh
14.登录mysql数据库
[root@db01 /opt/mysql-5.6.44/scripts]# mysql
Welcome to the MySQL monitor. Commands end with ; or g.
Your MySQL connection id is 1
Server version: 5.6.44 MySQL Community Server (GPL)
Copyright (c) 2000, 2019, Oracle and/or its affiliates. All rights reserved.
Oracle is a registered trademark of Oracle Corporation and/or its
affiliates. Other names may be trademarks of their respective
owners.
Type 'help;' or 'h' for help. Type 'c' to clear the current input statement.
mysql>
==============================================================================
#filebeat使用模块收集mysql慢日志
1.配置mysql错误日志和慢日志路径
编辑my.cnf
[root@db01 ~]# vim /etc/my.cnf
[mysqld]
slow_query_log=ON
slow_query_log_file=/opt/mysql/data/slow.log
long_query_time=1
2.重启mysql并制造慢日志
[root@db01 ~]# /etc/init.d/mysqld restart
3.慢日志制造语句
mysql<
select sleep(2) user,host from mysql.user ;
4.确认慢日志和错误日志确实有生成
[root@db01 ~]# mysql -e "show variables like '%slow_query_log%'"
+---------------------+----------------------------------+
| Variable_name | Value |
+---------------------+----------------------------------+
| slow_query_log | ON |
| slow_query_log_file | /opt/mysql/data/slow.log |
+---------------------+----------------------------------+
5.激活filebeat的mysql模块
[root@db01 ~]# filebeat modules enable mysql
6.配置mysql的模块
[root@db01 ~]# cat /etc/filebeat/modules.d/mysql.yml
- module: mysql
# Error logs
error:
enabled: true
var.paths: ["/opt/mysql/data/db01.err"]
# Slow logs
slowlog:
enabled: true
var.paths: ["/opt/mysql/data/slow.log"]
7.配置filebeat根据日志类型做判断
[root@db01 ~]# cat /etc/filebeat/filebeat.yml
filebeat.config.modules:
path: ${path.config}/modules.d/*.yml
reload.enabled: true
reload.period: 10s
output.elasticsearch:
hosts: ["10.0.0.51:9200"]
indices:
- index: "mysql_slow-%{[beat.version]}-%{+yyyy.MM}"
when.contains:
source: "/opt/mysql/data/slow.log"
- index: "mysql_error-%{[beat.version]}-%{+yyyy.MM}"
when.contains:
source: "/opt/mysql/data/db01.err"
setup.template.name: "mysql"
setup.template.pattern: "mysql-*"
setup.template.enabled: false
setup.template.overwrite: true
8.重启filebeat
[root@db01 ~]# systemctl restart filebeat
9.生成慢日志数据
mysql> select sleep(2) user,host from mysql.user ;
+------+-----------+
| user | host |
+------+-----------+
| 0 | 127.0.0.1 |
| 0 | ::1 |
| 0 | db01 |
| 0 | db01 |
| 0 | localhost |
| 0 | localhost |
+------+-----------+
6 rows in set (12.01 sec)
10.登录es-head插件查询和kibana添加查询
