Elastic Stack之FileBeat使用实战
作者：尹正杰
版权声明：原创作品，谢绝转载！否则将追究法律责任。
本篇博客数据流走向：FileBeat ===》logstash ===> elasticsearch。
一.安装FileBeat
1>.选择FileBeat版本（连接地址：https://www.elastic.co/downloads/beats）
2>.选择之前的发行版本
3>.选择FileBeat版本
4>.下载FileBeat（连接地址：https://www.elastic.co/downloads/past-releases/filebeat-5-6-12）
[root@node105 ~]# wget https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-5.6.12-x86_64.rpm --2019-03-11 22:21:04-- https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-5.6.12-x86_64.rpm Resolving artifacts.elastic.co (artifacts.elastic.co)... 151.101.230.222, 2a04:4e42:1a::734 Connecting to artifacts.elastic.co (artifacts.elastic.co)|151.101.230.222|:443... connected. HTTP request sent, awaiting response... 200 OK Length: 9224611 (8.8M) [application/octet-stream] Saving to: ‘filebeat-5.6.12-x86_64.rpm’ 100%[===========================================================================================================================================================>] 9,224,611 63.9KB/s in 2m 2s 2019-03-11 22:23:09 (74.0 KB/s) - ‘filebeat-5.6.12-x86_64.rpm’ saved [9224611/9224611] [root@node105 ~]#
二.使用FileBeat的案例实战
1>.安装FileBeat
[root@node105 ~]# [root@node105 ~]# ll total 139728 -rw-r--r--. 1 root root 9224611 Sep 19 03:10 filebeat-5.6.12-x86_64.rpm -rw-r--r--. 1 root root 28513410 Mar 4 23:29 GeoLite2-City.tar.gz -rw-r--r--. 1 root root 105333923 Sep 19 03:12 logstash-5.6.12.rpm [root@node105 ~]# [root@node105 ~]# [root@node105 ~]# rpm -ivh filebeat-5.6.12-x86_64.rpm warning: filebeat-5.6.12-x86_64.rpm: Header V4 RSA/SHA512 Signature, key ID d88e42b4: NOKEY Preparing... ################################# [100%] Updating / installing... 1:filebeat-5.6.12-1 ################################# [100%] [root@node105 ~]# [root@node105 ~]#
[root@node105 ~]# [root@node105 ~]# rpm -ql filebeat /etc/filebeat/filebeat.full.yml /etc/filebeat/filebeat.template-es2x.json /etc/filebeat/filebeat.template-es6x.json /etc/filebeat/filebeat.template.json /etc/filebeat/filebeat.yml /etc/init.d/filebeat /lib/systemd/system/filebeat.service /usr/bin/filebeat.sh /usr/share/filebeat/.build_hash.txt /usr/share/filebeat/NOTICE /usr/share/filebeat/README.md /usr/share/filebeat/bin/filebeat /usr/share/filebeat/bin/filebeat-god /usr/share/filebeat/module/apache2/access/config/access.yml /usr/share/filebeat/module/apache2/access/ingest/default.json /usr/share/filebeat/module/apache2/access/manifest.yml /usr/share/filebeat/module/apache2/error/config/error.yml /usr/share/filebeat/module/apache2/error/ingest/pipeline.json /usr/share/filebeat/module/apache2/error/manifest.yml /usr/share/filebeat/module/auditd/log/config/log.yml /usr/share/filebeat/module/auditd/log/ingest/pipeline.json /usr/share/filebeat/module/auditd/log/manifest.yml /usr/share/filebeat/module/mysql/error/config/error.yml /usr/share/filebeat/module/mysql/error/ingest/pipeline.json /usr/share/filebeat/module/mysql/error/manifest.yml /usr/share/filebeat/module/mysql/slowlog/config/slowlog.yml /usr/share/filebeat/module/mysql/slowlog/ingest/pipeline.json /usr/share/filebeat/module/mysql/slowlog/manifest.yml /usr/share/filebeat/module/nginx/access/config/nginx-access.yml /usr/share/filebeat/module/nginx/access/ingest/default.json /usr/share/filebeat/module/nginx/access/machine_learning/datafeed_low_request_rate.json /usr/share/filebeat/module/nginx/access/machine_learning/datafeed_remote_ip_request_rate.json /usr/share/filebeat/module/nginx/access/machine_learning/datafeed_remote_ip_url_count.json /usr/share/filebeat/module/nginx/access/machine_learning/datafeed_response_code.json /usr/share/filebeat/module/nginx/access/machine_learning/datafeed_visitor_rate.json /usr/share/filebeat/module/nginx/access/machine_learning/low_request_rate.json /usr/share/filebeat/module/nginx/access/machine_learning/remote_ip_request_rate.json /usr/share/filebeat/module/nginx/access/machine_learning/remote_ip_url_count.json /usr/share/filebeat/module/nginx/access/machine_learning/response_code.json /usr/share/filebeat/module/nginx/access/machine_learning/visitor_rate.json /usr/share/filebeat/module/nginx/access/manifest.yml /usr/share/filebeat/module/nginx/error/config/nginx-error.yml /usr/share/filebeat/module/nginx/error/ingest/pipeline.json /usr/share/filebeat/module/nginx/error/manifest.yml /usr/share/filebeat/module/system/auth/config/auth.yml /usr/share/filebeat/module/system/auth/ingest/pipeline.json /usr/share/filebeat/module/system/auth/manifest.yml /usr/share/filebeat/module/system/syslog/config/syslog.yml /usr/share/filebeat/module/system/syslog/ingest/pipeline.json /usr/share/filebeat/module/system/syslog/manifest.yml /usr/share/filebeat/scripts/import_dashboards /usr/share/filebeat/scripts/migrate_beat_config_1_x_to_5_0.py [root@node105 ~]#
2>.修改FileBeat的配置文件
[root@node105 ~]# [root@node105 ~]# [root@node105 ~]# cp /etc/filebeat/filebeat.yml{,.bak} [root@node105 ~]# [root@node105 ~]# ll /etc/filebeat/ total 124 -rw-r--r--. 1 root root 38827 Sep 11 04:06 filebeat.full.yml -rw-r--r--. 1 root root 25088 Sep 11 04:06 filebeat.template-es2x.json -rw-r--r--. 1 root root 20028 Sep 11 04:06 filebeat.template-es6x.json -rw-r--r--. 1 root root 20028 Sep 11 04:06 filebeat.template.json -rw-------. 1 root root 4196 Sep 11 04:06 filebeat.yml -rw-------. 1 root root 4196 Mar 11 22:26 filebeat.yml.bak [root@node105 ~]# [root@node105 ~]#
[root@node105 ~]# [root@node105 ~]# cat /etc/filebeat/filebeat.yml | egrep -v "^#|^$| #" filebeat.prospectors: - input_type: log paths: - /var/log/httpd/access_log output.logstash: hosts: ["node105.yinzhengjie.org.cn:5044"] [root@node105 ~]# [root@node105 ~]#
3>.修改Logstash的配置文件，配置输入插件为beats并启动（参考链接：https://www.elastic.co/guide/en/logstash/5.6/plugins-inputs-beats.html）
[root@node105 conf.d]# [root@node105 conf.d]# cat beats-filter-elasticsearch.conf input { beats { port =>5044 } } filter { grok { match => { "message" => "%{HTTPD_COMBINEDLOG}" } remove_field => "message" } date { match => ["timestamp","dd/MMM/YYYY:H:m:s Z"] remove_field => "timestamp" } geoip { source => "clientip" target => "geoip" database => "/etc/logstash/maxmind/GeoLite2-City.mmdb" } mutate { rename => { "agent" => "user_agent" } } } output { elasticsearch { hosts => ["http://node101.yinzhengjie.org.cn:9200/","http://node102.yinzhengjie.org.cn:9200/","http://node103.yinzhengjie.org.cn:9200/"] index => "logstash-%{+YYYY.MM.dd}" document_type => "httpd_access_logs" } } [root@node105 conf.d]# [root@node105 conf.d]# [root@node105 conf.d]# logstash -f beats-filter-elasticsearch.conf -t WARNING: Could not find logstash.yml which is typically located in $LS_HOME/config or /etc/logstash. You can specify the path using --path.settings. Continuing using the defaults Could not find log4j2 configuration at path /usr/share/logstash/config/log4j2.properties. Using default config which logs errors to the console Configuration OK [root@node105 conf.d]#
[root@node105 conf.d]# logstash -f beats-filter-elasticsearch.conf WARNING: Could not find logstash.yml which is typically located in $LS_HOME/config or /etc/logstash. You can specify the path using --path.settings. Continuing using the defaults Could not find log4j2 configuration at path /usr/share/logstash/config/log4j2.properties. Using default config which logs errors to the console [INFO ] 2019-03-11 22:43:28.925 [[main]<beats] Server - Starting server on port: 5044
[root@node105 ~]# [root@node105 ~]# ss -ntl | grep 5044 LISTEN 0 128 :::5044 :::* [root@node105 ~]#
4>.启动FileBeat服务
[root@node105 ~]# [root@node105 ~]# ss -ntl | grep 5044 LISTEN 0 128 :::5044 :::* [root@node105 ~]# [root@node105 ~]# [root@node105 ~]# [root@node105 ~]# systemctl start filebeat [root@node105 ~]# [root@node105 ~]# systemctl status filebeat ● filebeat.service - filebeat Loaded: loaded (/usr/lib/systemd/system/filebeat.service; disabled; vendor preset: disabled) Active: active (running) since Mon 2019-03-11 22:46:00 CST; 10s ago Docs: https://www.elastic.co/guide/en/beats/filebeat/current/index.html Main PID: 3839 (filebeat) CGroup: /system.slice/filebeat.service └─3839 /usr/share/filebeat/bin/filebeat -c /etc/filebeat/filebeat.yml -path.home /usr/share/filebeat -path.config /etc/filebeat -path.data /var/lib/filebeat -path.logs /var/log/filebe... Mar 11 22:46:00 node105.yinzhengjie.org.cn systemd[1]: Started filebeat. Mar 11 22:46:00 node105.yinzhengjie.org.cn systemd[1]: Starting filebeat... [root@node105 ~]# [root@node105 ~]#
[root@node105 ~]# [root@node105 ~]# ps -ef | grep filebeat | grep -v grep root 3839 1 0 22:46 ? 00:00:00 /usr/share/filebeat/bin/filebeat -c /etc/filebeat/filebeat.yml -path.home /usr/share/filebeat -path.config /etc/filebeat -path.data /var/lib/filebeat -path.logs /var/log/filebeat [root@node105 ~]# [root@node105 ~]#
5>.查看相应的日志信息
[root@node101 ~]# [root@node101 ~]# curl -X GET http://node101.yinzhengjie.org.cn:9200/logstash-*/_search?q=response:404 | jq . % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 100 7798 100 7798 0 0 542k 0 --:--:-- --:--:-- --:--:-- 585k { "took": 7, "timed_out": false, "_shards": { "total": 10, "successful": 10, "skipped": 0, "failed": 0 }, "hits": { "total": 117, "max_score": 2.14398, "hits": [ { "_index": "logstash-2019.03.11", "_type": "httpd_access_logs", "_id": "AWltN80kXxXllWpXYAEc", "_score": 2.14398, "_source": { "request": "/test60.html", "geoip": { "timezone": "Asia/Tokyo", "ip": "144.213.1.1", "latitude": 35.69, "country_name": "Japan", "country_code2": "JP", "continent_code": "AS", "country_code3": "JP", "location": { "lon": 139.69, "lat": 35.69 }, "longitude": 139.69 }, "offset": 25340, "auth": "-", "ident": "-", "input_type": "log", "verb": "GET", "source": "/var/log/httpd/access_log", "type": "log", "tags": [ "beats_input_codec_plain_applied" ], "referrer": ""-"", "@timestamp": "2019-03-11T14:02:27.000Z", "response": "404", "bytes": "209", "clientip": "144.213.1.1", "@version": "1", "beat": { "name": "node105.yinzhengjie.org.cn", "hostname": "node105.yinzhengjie.org.cn", "version": "5.6.12" }, "host": "node105.yinzhengjie.org.cn", "httpversion": "1.1", "user_agent": ""curl/7.29.0"" } }, { "_index": "logstash-2019.03.11", "_type": "httpd_access_logs", "_id": "AWltN80kXxXllWpXYAEd", "_score": 2.14398, "_source": { "request": "/test57.html", "geoip": { "ip": "175.91.1.1", "latitude": 34.7725, "country_name": "China", "country_code2": "CN", "continent_code": "AS", "country_code3": "CN", "location": { "lon": 113.7266, "lat": 34.7725 }, "longitude": 113.7266 }, "offset": 25438, "auth": "-", "ident": "-", "input_type": "log", "verb": "GET", "source": "/var/log/httpd/access_log", "type": "log", "tags": [ "beats_input_codec_plain_applied" ], "referrer": ""-"", "@timestamp": "2019-03-11T14:02:28.000Z", "response": "404", "bytes": "209", "clientip": "175.91.1.1", "@version": "1", "beat": { "name": "node105.yinzhengjie.org.cn", "hostname": "node105.yinzhengjie.org.cn", "version": "5.6.12" }, "host": "node105.yinzhengjie.org.cn", "httpversion": "1.1", "user_agent": ""curl/7.29.0"" } }, { "_index": "logstash-2019.03.11", "_type": "httpd_access_logs", "_id": "AWltEA3lXxXllWpXYACI", "_score": 2.14398, "_source": { "request": "/test52.html", "geoip": { "timezone": "Europe/Paris", "ip": "82.234.1.1", "latitude": 48.6942, "continent_code": "EU", "city_name": "Brunoy", "country_name": "France", "country_code2": "FR", "country_code3": "FR", "region_name": "Essonne", "location": { "lon": 2.4922, "lat": 48.6942 }, "postal_code": "91800", "region_code": "91", "longitude": 2.4922 }, "auth": "-", "ident": "-", "verb": "GET", "path": "/var/log/httpd/access_log", "referrer": ""-"", "@timestamp": "2019-03-11T14:02:41.000Z", "response": "404", "bytes": "209", "clientip": "82.234.1.1", "@version": "1", "host": "0.0.0.0", "httpversion": "1.1", "user_agent": ""curl/7.29.0"" } }, { "_index": "logstash-2019.03.11", "_type": "httpd_access_logs", "_id": "AWltEAIs3WCT5NaOiwEi", "_score": 2.14398, "_source": { "request": "/test53.html", "geoip": { "city_name": "Shanghai", "timezone": "Asia/Shanghai", "ip": "180.170.1.1", "latitude": 31.0449, "country_name": "China", "country_code2": "CN", "continent_code": "AS", "country_code3": "CN", "region_name": "Shanghai", "location": { "lon": 121.4012, "lat": 31.0449 }, "region_code": "SH", "longitude": 121.4012 }, "auth": "-", "ident": "-", "verb": "GET", "path": "/var/log/httpd/access_log", "referrer": ""-"", "@timestamp": "2019-03-11T14:02:38.000Z", "response": "404", "bytes": "209", "clientip": "180.170.1.1", "@version": "1", "host": "0.0.0.0", "httpversion": "1.1", "user_agent": ""curl/7.29.0"" } }, { "_index": "logstash-2019.03.11", "_type": "httpd_access_logs", "_id": "AWltD9ccXxXllWpXYACF", "_score": 2.14398, "_source": { "request": "/test60.html", "geoip": { "timezone": "Asia/Tokyo", "ip": "144.213.1.1", "latitude": 35.69, "country_name": "Japan", "country_code2": "JP", "continent_code": "AS", "country_code3": "JP", "location": { "lon": 139.69, "lat": 35.69 }, "longitude": 139.69 }, "auth": "-", "ident": "-", "verb": "GET", "path": "/var/log/httpd/access_log", "referrer": ""-"", "@timestamp": "2019-03-11T14:02:27.000Z", "response": "404", "bytes": "209", "clientip": "144.213.1.1", "@version": "1", "host": "0.0.0.0", "httpversion": "1.1", "user_agent": ""curl/7.29.0"" } }, { "_index": "logstash-2019.03.11", "_type": "httpd_access_logs", "_id": "AWltEBmksru-A5a8RIhh", "_score": 2.14398, "_source": { "request": "/test52.html", "geoip": { "timezone": "Africa/Accra", "ip": "45.209.1.1", "latitude": 8, "country_name": "Ghana", "country_code2": "GH", "continent_code": "AF", "country_code3": "GH", "location": { "lon": -2, "lat": 8 }, "longitude": -2 }, "auth": "-", "ident": "-", "verb": "GET", "path": "/var/log/httpd/access_log", "referrer": ""-"", "@timestamp": "2019-03-11T14:02:44.000Z", "response": "404", "bytes": "209", "clientip": "45.209.1.1", "@version": "1", "host": "0.0.0.0", "httpversion": "1.1", "user_agent": ""curl/7.29.0"" } }, { "_index": "logstash-2019.03.11", "_type": "httpd_access_logs", "_id": "AWltEMH9XxXllWpXYACa", "_score": 2.14398, "_source": { "request": "/test53.html", "geoip": { "timezone": "Europe/Rome", "ip": "94.164.1.1", "latitude": 42.5245, "continent_code": "EU", "city_name": "Piansano", "country_name": "Italy", "country_code2": "IT", "country_code3": "IT", "region_name": "Provincia di Viterbo", "location": { "lon": 11.8298, "lat": 42.5245 }, "postal_code": "01010", "region_code": "VT", "longitude": 11.8298 }, "auth": "-", "ident": "-", "verb": "GET", "path": "/var/log/httpd/access_log", "referrer": ""-"", "@timestamp": "2019-03-11T14:03:28.000Z", "response": "404", "bytes": "209", "clientip": "94.164.1.1", "@version": "1", "host": "0.0.0.0", "httpversion": "1.1", "user_agent": ""curl/7.29.0"" } }, { "_index": "logstash-2019.03.11", "_type": "httpd_access_logs", "_id": "AWltOqroXxXllWpXYAF7", "_score": 2.14398, "_source": { "request": "/test53.html", "geoip": { "timezone": "Asia/Ho_Chi_Minh", "ip": "171.247.1.1", "latitude": 16, "country_name": "Vietnam", "country_code2": "VN", "continent_code": "AS", "country_code3": "VN", "location": { "lon": 106, "lat": 16 }, "longitude": 106 }, "offset": 40046, "auth": "-", "ident": "-", "input_type": "log", "verb": "GET", "source": "/var/log/httpd/access_log", "type": "log", "tags": [ "beats_input_codec_plain_applied" ], "referrer": ""-"", "@timestamp": "2019-03-11T14:49:10.000Z", "response": "404", "bytes": "209", "clientip": "171.247.1.1", "@version": "1", "beat": { "name": "node105.yinzhengjie.org.cn", "hostname": "node105.yinzhengjie.org.cn", "version": "5.6.12" }, "host": "node105.yinzhengjie.org.cn", "httpversion": "1.1", "user_agent": ""curl/7.29.0"" } }, { "_index": "logstash-2019.03.11", "_type": "httpd_access_logs", "_id": "AWltOpdbsru-A5a8RIib", "_score": 2.14398, "_source": { "request": "/test54.html", "geoip": { "ip": "177.3.1.1", "latitude": -22.8305, "country_name": "Brazil", "country_code2": "BR", "continent_code": "SA", "country_code3": "BR", "location": { "lon": -43.2192, "lat": -22.8305 }, "longitude": -43.2192 }, "offset": 39757, "auth": "-", "ident": "-", "input_type": "log", "verb": "GET", "source": "/var/log/httpd/access_log", "type": "log", "tags": [ "beats_input_codec_plain_applied" ], "referrer": ""-"", "@timestamp": "2019-03-11T14:49:07.000Z", "response": "404", "bytes": "209", "clientip": "177.3.1.1", "@version": "1", "beat": { "name": "node105.yinzhengjie.org.cn", "hostname": "node105.yinzhengjie.org.cn", "version": "5.6.12" }, "host": "node105.yinzhengjie.org.cn", "httpversion": "1.1", "user_agent": ""curl/7.29.0"" } }, { "_index": "logstash-2019.03.11", "_type": "httpd_access_logs", "_id": "AWltO0cmsru-A5a8RIi5", "_score": 2.14398, "_source": { "request": "/test56.html", "geoip": { "city_name": "Taipei", "timezone": "Asia/Taipei", "ip": "61.228.1.1", "latitude": 25.0478, "country_name": "Taiwan", "country_code2": "TW", "continent_code": "AS", "country_code3": "TW", "region_name": "Taipei City", "location": { "lon": 121.5318, "lat": 25.0478 }, "region_code": "TPE", "longitude": 121.5318 }, "offset": 43986, "auth": "-", "ident": "-", "input_type": "log", "verb": "GET", "source": "/var/log/httpd/access_log", "type": "log", "tags": [ "beats_input_codec_plain_applied" ], "referrer": ""-"", "@timestamp": "2019-03-11T14:49:51.000Z", "response": "404", "bytes": "209", "clientip": "61.228.1.1", "@version": "1", "beat": { "name": "node105.yinzhengjie.org.cn", "hostname": "node105.yinzhengjie.org.cn", "version": "5.6.12" }, "host": "node105.yinzhengjie.org.cn", "httpversion": "1.1", "user_agent": ""curl/7.29.0"" } } ] } } [root@node101 ~]# [root@node101 ~]#