Filebeat使用快速入门篇
作者：尹正杰
版权声明：原创作品，谢绝转载！否则将追究法律责任。
一.部署filebeat
博主推荐阅读: https://www.elastic.co/guide/en/beats/filebeat/6.8/filebeat-installation.html
[root@es103.yinzhengjie.com ~]# ls #将下载的filebeat软件包上传到服务器上 filebeat-6.8.9-amd64.deb [root@es103.yinzhengjie.com ~]# [root@es103.yinzhengjie.com ~]# dpkg -i filebeat-6.8.9-amd64.deb #使用dpkg工具安装filebeat Selecting previously unselected package filebeat. (Reading database ... 156175 files and directories currently installed.) Preparing to unpack filebeat-6.8.9-amd64.deb ... Unpacking filebeat (6.8.9) ... Setting up filebeat (6.8.9) ... Processing triggers for systemd (237-3ubuntu10.38) ... Processing triggers for ureadahead (0.100.0-21) ... [root@es103.yinzhengjie.com ~]#
二.将数据直接写入到elasticsearch集群
1>.编写配置文件
[root@es103.yinzhengjie.com ~]# vim /etc/filebeat/filebeat.yml [root@es103.yinzhengjie.com ~]# [root@es103.yinzhengjie.com ~]# egrep -v "^*#|^$" /etc/filebeat/filebeat.yml filebeat.inputs: - type: log enabled: true paths: - /var/log/syslog fields: host: "172.200.5.103" type: "filebeat-syslog-172-200-5-103" app: "ubuntu-syslog" filebeat.config.modules: path: ${path.config}/modules.d/*.yml reload.enabled: false setup.template.enabled: false setup.template.settings: index.number_of_shards: 3 setup.kibana: output.elasticsearch: hosts: ["http://es101.yinzhengjie.com:9200","http://es102.yinzhengjie.com:9200","http://es103.yinzhengjie.com:9200"] index: "filebeat-%{[beat.version]}-syslog-172-200-5-103-%{+yyyy.MM.dd}" processors: - add_host_metadata: ~ - add_cloud_metadata: ~ [root@es103.yinzhengjie.com ~]# [root@es103.yinzhengjie.com ~]#
2>.启动filebeat服务
[root@es103.yinzhengjie.com ~]# systemctl start filebeat [root@es103.yinzhengjie.com ~]# [root@es103.yinzhengjie.com ~]# systemctl status filebeat ● filebeat.service - Filebeat sends log files to Logstash or directly to Elasticsearch. Loaded: loaded (/lib/systemd/system/filebeat.service; disabled; vendor preset: enabled) Active: active (running) since Fri 2020-06-19 00:50:05 UTC; 8min ago Docs: https://www.elastic.co/products/beats/filebeat Main PID: 3230 (filebeat) Tasks: 12 (limit: 3413) CGroup: /system.slice/filebeat.service └─3230 /usr/share/filebeat/bin/filebeat -c /etc/filebeat/filebeat.yml -path.home /usr/share/filebeat -path.config /etc/filebeat -path.data /var/lib/filebeat -path.logs /var/log/f Jun 19 00:50:05 es103.yinzhengjie.com systemd[1]: Started Filebeat sends log files to Logstash or directly to Elasticsearch.. [root@es103.yinzhengjie.com ~]#
[root@es103.yinzhengjie.com ~]# ps -ef | grep filebeat root 3230 1 0 00:50 ? 00:00:00 /usr/share/filebeat/bin/filebeat -c /etc/filebeat/filebeat.yml -path.home /usr/share/filebeat -path.config /etc/filebeat -path.data /var/li b/filebeat -path.logs /var/log/filebeatroot 3278 2449 0 01:00 pts/1 00:00:00 grep --color=auto filebeat [root@es103.yinzhengjie.com ~]#
[root@es103.yinzhengjie.com ~]# tail -100f /var/log/filebeat/filebeat 2020-06-19T00:50:05.420Z INFO instance/beat.go:611 Home path: [/usr/share/filebeat] Config path: [/etc/filebeat] Data path: [/var/lib/filebeat] Logs path: [/var/log/filebeat] 2020-06-19T00:50:05.421Z INFO instance/beat.go:618 Beat UUID: 8f27e999-8ad7-4ee0-9b60-9d9331a7ba43 2020-06-19T00:50:05.422Z INFO [seccomp] seccomp/seccomp.go:116 Syscall filter successfully installed 2020-06-19T00:50:05.423Z INFO [beat] instance/beat.go:931 Beat info {"system_info": {"beat": {"path": {"config": "/etc/filebeat", "data": "/var/lib/filebeat", "home": "/ usr/share/filebeat", "logs": "/var/log/filebeat"}, "type": "filebeat", "uuid": "8f27e999-8ad7-4ee0-9b60-9d9331a7ba43"}}}2020-06-19T00:50:05.423Z INFO [beat] instance/beat.go:940 Build info {"system_info": {"build": {"commit": "75d698ad90ade7d497d9b3584654a3a2617e2f73", "libbeat": "6.8.9", "time": "2020-05-04T16:04:24.000Z", "version": "6.8.9"}}}2020-06-19T00:50:05.423Z INFO [beat] instance/beat.go:943 Go runtime info {"system_info": {"go": {"os":"linux","arch":"amd64","max_procs":2,"version":"go1.10.8"}}} 2020-06-19T00:50:05.425Z INFO [beat] instance/beat.go:947 Host info {"system_info": {"host": {"architecture":"x86_64","boot_time":"2020-06-18T23:55:39Z","containerized": false,"name":"es103.yinzhengjie.com","ip":["127.0.0.1/8","::1/128","172.200.5.103/21","fe80::20c:29ff:fe06:42e6/64"],"kernel_version":"4.15.0-106-generic","mac":["00:0c:29:06:42:e6"],"os":{"family":"debian","platform":"ubuntu","name":"Ubuntu","version":"18.04.3 LTS (Bionic Beaver)","major":18,"minor":4,"patch":3,"codename":"bionic"},"timezone":"UTC","timezone_offset_sec":0,"id":"8e0a6eabcd8c40f19da6b7f090c4cd2a"}}}2020-06-19T00:50:05.426Z INFO [beat] instance/beat.go:976 Process info {"system_info": {"process": {"capabilities": {"inheritable":null,"permitted":["chown","dac_override", "dac_read_search","fowner","fsetid","kill","setgid","setuid","setpcap","linux_immutable","net_bind_service","net_broadcast","net_admin","net_raw","ipc_lock","ipc_owner","sys_module","sys_rawio","sys_chroot","sys_ptrace","sys_pacct","sys_admin","sys_boot","sys_nice","sys_resource","sys_time","sys_tty_config","mknod","lease","audit_write","audit_control","setfcap","mac_override","mac_admin","syslog","wake_alarm","block_suspend","audit_read"],"effective":["chown","dac_override","dac_read_search","fowner","fsetid","kill","setgid","setuid","setpcap","linux_immutable","net_bind_service","net_broadcast","net_admin","net_raw","ipc_lock","ipc_owner","sys_module","sys_rawio","sys_chroot","sys_ptrace","sys_pacct","sys_admin","sys_boot","sys_nice","sys_resource","sys_time","sys_tty_config","mknod","lease","audit_write","audit_control","setfcap","mac_override","mac_admin","syslog","wake_alarm","block_suspend","audit_read"],"bounding":["chown","dac_override","dac_read_search","fowner","fsetid","kill","setgid","setuid","setpcap","linux_immutable","net_bind_service","net_broadcast","net_admin","net_raw","ipc_lock","ipc_owner","sys_module","sys_rawio","sys_chroot","sys_ptrace","sys_pacct","sys_admin","sys_boot","sys_nice","sys_resource","sys_time","sys_tty_config","mknod","lease","audit_write","audit_control","setfcap","mac_override","mac_admin","syslog","wake_alarm","block_suspend","audit_read"],"ambient":null}, "cwd": "/", "exe": "/usr/share/filebeat/bin/filebeat", "name": "filebeat", "pid": 3230, "ppid": 1, "seccomp": {"mode":"filter","no_new_privs":true}, "start_time": "2020-06-19T00:50:04.300Z"}}}2020-06-19T00:50:05.426Z INFO instance/beat.go:280 Setup Beat: filebeat; Version: 6.8.9 2020-06-19T00:50:05.427Z INFO elasticsearch/client.go:164 Elasticsearch url: http://es101.yinzhengjie.com:9200 2020-06-19T00:50:05.427Z INFO elasticsearch/client.go:164 Elasticsearch url: http://es102.yinzhengjie.com:9200 2020-06-19T00:50:05.427Z INFO elasticsearch/client.go:164 Elasticsearch url: http://es103.yinzhengjie.com:9200 2020-06-19T00:50:05.428Z INFO [publisher] pipeline/module.go:110 Beat name: es103.yinzhengjie.com 2020-06-19T00:50:05.450Z INFO instance/beat.go:402 filebeat start running. 2020-06-19T00:50:05.450Z INFO registrar/registrar.go:97 No registry file found under: /var/lib/filebeat/registry. Creating a new registry file. 2020-06-19T00:50:05.469Z INFO [monitoring] log/log.go:117 Starting metrics logging every 30s 2020-06-19T00:50:05.512Z INFO registrar/registrar.go:134 Loading registrar data from /var/lib/filebeat/registry 2020-06-19T00:50:05.512Z INFO registrar/registrar.go:141 States Loaded from registrar: 0 2020-06-19T00:50:05.512Z INFO crawler/crawler.go:72 Loading Inputs: 1 2020-06-19T00:50:05.513Z INFO log/input.go:148 Configured paths: [/var/log/syslog] 2020-06-19T00:50:05.513Z INFO input/input.go:114 Starting input of type: log; ID: 5479846534217438201 2020-06-19T00:50:05.545Z INFO crawler/crawler.go:106 Loading and starting Inputs completed. Enabled inputs: 1 2020-06-19T00:50:05.545Z INFO cfgfile/reload.go:150 Config reloader started 2020-06-19T00:50:05.545Z INFO cfgfile/reload.go:205 Loading of config files completed. 2020-06-19T00:50:05.554Z INFO log/harvester.go:255 Harvester started for file: /var/log/syslog 2020-06-19T00:50:08.430Z INFO add_cloud_metadata/add_cloud_metadata.go:340 add_cloud_metadata: hosting provider type not detected. 2020-06-19T00:50:08.552Z INFO pipeline/output.go:95 Connecting to backoff(elasticsearch(http://es103.yinzhengjie.com:9200)) 2020-06-19T00:50:08.552Z INFO pipeline/output.go:95 Connecting to backoff(elasticsearch(http://es101.yinzhengjie.com:9200)) 2020-06-19T00:50:08.552Z INFO pipeline/output.go:95 Connecting to backoff(elasticsearch(http://es102.yinzhengjie.com:9200)) 2020-06-19T00:50:08.600Z INFO elasticsearch/client.go:739 Attempting to connect to Elasticsearch version 6.8.9 2020-06-19T00:50:08.624Z INFO pipeline/output.go:105 Connection to backoff(elasticsearch(http://es103.yinzhengjie.com:9200)) established 2020-06-19T00:50:08.635Z INFO elasticsearch/client.go:739 Attempting to connect to Elasticsearch version 6.8.9 2020-06-19T00:50:08.649Z INFO elasticsearch/client.go:739 Attempting to connect to Elasticsearch version 6.8.9 2020-06-19T00:50:08.661Z INFO pipeline/output.go:105 Connection to backoff(elasticsearch(http://es102.yinzhengjie.com:9200)) established 2020-06-19T00:50:08.692Z INFO pipeline/output.go:105 Connection to backoff(elasticsearch(http://es101.yinzhengjie.com:9200)) established 2020-06-19T00:50:35.473Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":290,"time":{"ms" :295}},"total":{"ticks":420,"time":{"ms":430},"value":420},"user":{"ticks":130,"time":{"ms":135}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":10},"info":{"ephemeral_id":"e2f58be5-c311-422f-acf0-540d8acd8950","uptime":{"ms":30108}},"memstats":{"gc_next":21630224,"memory_alloc":11569504,"memory_total":29177024,"rss":38166528}},"filebeat":{"events":{"added":2305,"done":2305},"harvester":{"open_files":1,"running":1,"started":1}},"libbeat":{"config":{"module":{"running":0},"reloads":1},"output":{"events":{"acked":2304,"batches":47,"total":2304},"read":{"bytes":34315},"type":"elasticsearch","write":{"bytes":1912260}},"pipeline":{"clients":1,"events":{"active":0,"filtered":1,"published":2304,"retry":150,"total":2305},"queue":{"acked":2304}}},"registrar":{"states":{"current":1,"update":2305},"writes":{"success":17,"total":17}},"system":{"cpu":{"cores":2},"load":{"1":0.01,"15":0.05,"5":0.03,"norm":{"1":0.005,"15":0.025,"5":0.015}}}}}}2020-06-19T00:51:05.473Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":300,"time":{"ms" :7}},"total":{"ticks":430,"time":{"ms":8},"value":430},"user":{"ticks":130,"time":{"ms":1}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":10},"info":{"ephemeral_id":"e2f58be5-c311-422f-acf0-540d8acd8950","uptime":{"ms":60106}},"memstats":{"gc_next":21630224,"memory_alloc":11888504,"memory_total":29496024}},"filebeat":{"harvester":{"open_files":1,"running":1}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":1}},"system":{"load":{"1":0.07,"15":0.05,"5":0.04,"norm":{"1":0.035,"15":0.025,"5":0.02}}}}}}2020-06-19T00:51:35.472Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":300,"time":{"ms" :6}},"total":{"ticks":430,"time":{"ms":6},"value":430},"user":{"ticks":130}},"handles":{"limit":{"hard":4096,"soft":1024},"open":10},"info":{"ephemeral_id":"e2f58be5-c311-422f-acf0-540d8acd8950","uptime":{"ms":90107}},"memstats":{"gc_next":21630224,"memory_alloc":12338072,"memory_total":29945592}},"filebeat":{"harvester":{"open_files":1,"running":1}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":1}},"system":{"load":{"1":0.04,"15":0.05,"5":0.03,"norm":{"1":0.02,"15":0.025,"5":0.015}}}}}}2020-06-19T00:52:05.472Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":310,"time":{"ms" :9}},"total":{"ticks":440,"time":{"ms":10},"value":440},"user":{"ticks":130,"time":{"ms":1}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":10},"info":{"ephemeral_id":"e2f58be5-c311-422f-acf0-540d8acd8950","uptime":{"ms":120107}},"memstats":{"gc_next":21630224,"memory_alloc":12474344,"memory_total":30081864}},"filebeat":{"harvester":{"open_files":1,"running":1}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":1}},"system":{"load":{"1":0.02,"15":0.04,"5":0.03,"norm":{"1":0.01,"15":0.02,"5":0.015}}}}}}2020-06-19T00:52:35.473Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":320,"time":{"ms" :3}},"total":{"ticks":460,"time":{"ms":8},"value":460},"user":{"ticks":140,"time":{"ms":5}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":10},"info":{"ephemeral_id":"e2f58be5-c311-422f-acf0-540d8acd8950","uptime":{"ms":150107}},"memstats":{"gc_next":5088272,"memory_alloc":2631696,"memory_total":30450256}},"filebeat":{"harvester":{"open_files":1,"running":1}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":1}},"system":{"load":{"1":0.01,"15":0.04,"5":0.03,"norm":{"1":0.005,"15":0.02,"5":0.015}}}}}}2020-06-19T00:53:05.472Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":320,"time":{"ms" :7}},"total":{"ticks":460,"time":{"ms":8},"value":460},"user":{"ticks":140,"time":{"ms":1}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":10},"info":{"ephemeral_id":"e2f58be5-c311-422f-acf0-540d8acd8950","uptime":{"ms":180106}},"memstats":{"gc_next":5088272,"memory_alloc":2856688,"memory_total":30675248}},"filebeat":{"harvester":{"open_files":1,"running":1}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":1}},"system":{"load":{"1":0.01,"15":0.04,"5":0.02,"norm":{"1":0.005,"15":0.02,"5":0.01}}}}}}2020-06-19T00:53:35.473Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":330,"time":{"ms" :6}},"total":{"ticks":470,"time":{"ms":6},"value":470},"user":{"ticks":140}},"handles":{"limit":{"hard":4096,"soft":1024},"open":10},"info":{"ephemeral_id":"e2f58be5-c311-422f-acf0-540d8acd8950","uptime":{"ms":210108}},"memstats":{"gc_next":5088272,"memory_alloc":3152064,"memory_total":30970624}},"filebeat":{"harvester":{"open_files":1,"running":1}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":1}},"system":{"load":{"1":0,"15":0.04,"5":0.02,"norm":{"1":0,"15":0.02,"5":0.01}}}}}}2020-06-19T00:54:05.476Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":340,"time":{"ms" :9}},"total":{"ticks":480,"time":{"ms":11},"value":480},"user":{"ticks":140,"time":{"ms":2}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":10},"info":{"ephemeral_id":"e2f58be5-c311-422f-acf0-540d8acd8950","uptime":{"ms":240108}},"memstats":{"gc_next":5088272,"memory_alloc":3440496,"memory_total":31259056}},"filebeat":{"harvester":{"open_files":1,"running":1}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":1}},"system":{"load":{"1":0,"15":0.03,"5":0.02,"norm":{"1":0,"15":0.015,"5":0.01}}}}}}2020-06-19T00:54:35.475Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":350,"time":{"ms" :11}},"total":{"ticks":500,"time":{"ms":20},"value":500},"user":{"ticks":150,"time":{"ms":9}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":10},"info":{"ephemeral_id":"e2f58be5-c311-422f-acf0-540d8acd8950","uptime":{"ms":270107}},"memstats":{"gc_next":5088272,"memory_alloc":2555704,"memory_total":31551968}},"filebeat":{"harvester":{"open_files":1,"running":1}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":1}},"system":{"load":{"1":0,"15":0.03,"5":0.01,"norm":{"1":0,"15":0.015,"5":0.005}}}}}}2020-06-19T00:55:05.478Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":360,"time":{"ms" :9}},"total":{"ticks":510,"time":{"ms":13},"value":510},"user":{"ticks":150,"time":{"ms":4}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":10},"info":{"ephemeral_id":"e2f58be5-c311-422f-acf0-540d8acd8950","uptime":{"ms":300108}},"memstats":{"gc_next":5088272,"memory_alloc":2854144,"memory_total":31850408}},"filebeat":{"harvester":{"open_files":1,"running":1}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":1}},"system":{"load":{"1":0,"15":0.03,"5":0.01,"norm":{"1":0,"15":0.015,"5":0.005}}}}}}2020-06-19T00:55:13.586Z INFO log/harvester.go:280 File is inactive: /var/log/syslog. Closing because close_inactive of 5m0s reached. 2020-06-19T00:55:35.474Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":370,"time":{"ms" :11}},"total":{"ticks":530,"time":{"ms":13},"value":530},"user":{"ticks":160,"time":{"ms":2}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":9},"info":{"ephemeral_id":"e2f58be5-c311-422f-acf0-540d8acd8950","uptime":{"ms":330109}},"memstats":{"gc_next":5088272,"memory_alloc":3149872,"memory_total":32146136}},"filebeat":{"events":{"added":1,"done":1},"harvester":{"closed":1,"open_files":0,"running":0}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0,"filtered":1,"total":1}}},"registrar":{"states":{"current":1,"update":1},"writes":{"success":1,"total":1}},"system":{"load":{"1":0,"15":0.02,"5":0.01,"norm":{"1":0,"15":0.01,"5":0.005}}}}}}2020-06-19T00:56:05.472Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":370,"time":{"ms" :5}},"total":{"ticks":530,"time":{"ms":6},"value":530},"user":{"ticks":160,"time":{"ms":1}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":9},"info":{"ephemeral_id":"e2f58be5-c311-422f-acf0-540d8acd8950","uptime":{"ms":360107}},"memstats":{"gc_next":5088272,"memory_alloc":3444784,"memory_total":32441048}},"filebeat":{"harvester":{"open_files":0,"running":0}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":1}},"system":{"load":{"1":0,"15":0.02,"5":0,"norm":{"1":0,"15":0.01,"5":0}}}}}}2020-06-19T00:56:35.472Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":380,"time":{"ms" :8}},"total":{"ticks":540,"time":{"ms":9},"value":540},"user":{"ticks":160,"time":{"ms":1}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":9},"info":{"ephemeral_id":"e2f58be5-c311-422f-acf0-540d8acd8950","uptime":{"ms":390106}},"memstats":{"gc_next":5040384,"memory_alloc":2529904,"memory_total":32731192}},"filebeat":{"harvester":{"open_files":0,"running":0}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":1}},"system":{"load":{"1":0.08,"15":0.02,"5":0.02,"norm":{"1":0.04,"15":0.01,"5":0.01}}}}}}2020-06-19T00:57:05.473Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":380,"time":{"ms" :3}},"total":{"ticks":540,"time":{"ms":8},"value":540},"user":{"ticks":160,"time":{"ms":5}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":9},"info":{"ephemeral_id":"e2f58be5-c311-422f-acf0-540d8acd8950","uptime":{"ms":420107}},"memstats":{"gc_next":5040384,"memory_alloc":2818552,"memory_total":33019840}},"filebeat":{"harvester":{"open_files":0,"running":0}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":1}},"system":{"load":{"1":0.05,"15":0.02,"5":0.02,"norm":{"1":0.025,"15":0.01,"5":0.01}}}}}}2020-06-19T00:57:35.472Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":400,"time":{"ms" :12}},"total":{"ticks":560,"time":{"ms":12},"value":560},"user":{"ticks":160}},"handles":{"limit":{"hard":4096,"soft":1024},"open":9},"info":{"ephemeral_id":"e2f58be5-c311-422f-acf0-540d8acd8950","uptime":{"ms":450107}},"memstats":{"gc_next":5040384,"memory_alloc":3197768,"memory_total":33399056,"rss":-9244672}},"filebeat":{"harvester":{"open_files":0,"running":0}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":1}},"system":{"load":{"1":0.03,"15":0.02,"5":0.01,"norm":{"1":0.015,"15":0.01,"5":0.005}}}}}}2020-06-19T00:58:05.477Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":410,"time":{"ms" :10}},"total":{"ticks":570,"time":{"ms":10},"value":570},"user":{"ticks":160}},"handles":{"limit":{"hard":4096,"soft":1024},"open":9},"info":{"ephemeral_id":"e2f58be5-c311-422f-acf0-540d8acd8950","uptime":{"ms":480107}},"memstats":{"gc_next":5040384,"memory_alloc":3402056,"memory_total":33603344}},"filebeat":{"harvester":{"open_files":0,"running":0}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":1}},"system":{"load":{"1":0.02,"15":0.01,"5":0.01,"norm":{"1":0.01,"15":0.005,"5":0.005}}}}}}2020-06-19T00:58:35.472Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":410,"time":{"ms" :1}},"total":{"ticks":580,"time":{"ms":9},"value":580},"user":{"ticks":170,"time":{"ms":8}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":9},"info":{"ephemeral_id":"e2f58be5-c311-422f-acf0-540d8acd8950","uptime":{"ms":510107}},"memstats":{"gc_next":5040480,"memory_alloc":2694632,"memory_total":34061176}},"filebeat":{"harvester":{"open_files":0,"running":0}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":1}},"system":{"load":{"1":0.01,"15":0.01,"5":0.01,"norm":{"1":0.005,"15":0.005,"5":0.005}}}}}}2020-06-19T00:59:05.473Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":410,"time":{"ms" :2}},"total":{"ticks":580,"time":{"ms":6},"value":580},"user":{"ticks":170,"time":{"ms":4}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":9},"info":{"ephemeral_id":"e2f58be5-c311-422f-acf0-540d8acd8950","uptime":{"ms":540107}},"memstats":{"gc_next":5040480,"memory_alloc":2822952,"memory_total":34189496}},"filebeat":{"harvester":{"open_files":0,"running":0}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":1}},"system":{"load":{"1":0,"15":0.01,"5":0,"norm":{"1":0,"15":0.005,"5":0}}}}}}2020-06-19T00:59:35.472Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":410,"time":{"ms" :4}},"total":{"ticks":580,"time":{"ms":4},"value":580},"user":{"ticks":170}},"handles":{"limit":{"hard":4096,"soft":1024},"open":9},"info":{"ephemeral_id":"e2f58be5-c311-422f-acf0-540d8acd8950","uptime":{"ms":570108}},"memstats":{"gc_next":5040480,"memory_alloc":3105160,"memory_total":34471704}},"filebeat":{"harvester":{"open_files":0,"running":0}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":1}},"system":{"load":{"1":0.07,"15":0.01,"5":0.02,"norm":{"1":0.035,"15":0.005,"5":0.01}}}}}}2020-06-19T01:00:05.473Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":420,"time":{"ms" :5}},"total":{"ticks":590,"time":{"ms":5},"value":590},"user":{"ticks":170}},"handles":{"limit":{"hard":4096,"soft":1024},"open":9},"info":{"ephemeral_id":"e2f58be5-c311-422f-acf0-540d8acd8950","uptime":{"ms":600106}},"memstats":{"gc_next":5040480,"memory_alloc":3401368,"memory_total":34767912,"rss":-663552}},"filebeat":{"harvester":{"open_files":0,"running":0}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":1}},"system":{"load":{"1":0.04,"15":0.01,"5":0.02,"norm":{"1":0.02,"15":0.005,"5":0.01}}}}}}2020-06-19T01:00:35.477Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":420,"time":{"ms" :7}},"total":{"ticks":600,"time":{"ms":16},"value":600},"user":{"ticks":180,"time":{"ms":9}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":9},"info":{"ephemeral_id":"e2f58be5-c311-422f-acf0-540d8acd8950","uptime":{"ms":630106}},"memstats":{"gc_next":5040416,"memory_alloc":2522272,"memory_total":35051288}},"filebeat":{"harvester":{"open_files":0,"running":0}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":1}},"system":{"load":{"1":0.02,"15":0.01,"5":0.01,"norm":{"1":0.01,"15":0.005,"5":0.005}}}}}}2020-06-19T01:01:05.472Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":430,"time":{"ms" :5}},"total":{"ticks":620,"time":{"ms":9},"value":620},"user":{"ticks":190,"time":{"ms":4}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":9},"info":{"ephemeral_id":"e2f58be5-c311-422f-acf0-540d8acd8950","uptime":{"ms":660107}},"memstats":{"gc_next":5040416,"memory_alloc":2905880,"memory_total":35434896}},"filebeat":{"harvester":{"open_files":0,"running":0}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":1}},"system":{"load":{"1":0.01,"15":0,"5":0.01,"norm":{"1":0.005,"15":0,"5":0.005}}}}}}2020-06-19T01:01:35.479Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":430,"time":{"ms" :5}},"total":{"ticks":620,"time":{"ms":10},"value":620},"user":{"ticks":190,"time":{"ms":5}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":9},"info":{"ephemeral_id":"e2f58be5-c311-422f-acf0-540d8acd8950","uptime":{"ms":690107}},"memstats":{"gc_next":5040416,"memory_alloc":3106776,"memory_total":35635792}},"filebeat":{"harvester":{"open_files":0,"running":0}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":1}},"system":{"load":{"1":0.01,"15":0,"5":0.01,"norm":{"1":0.005,"15":0,"5":0.005}}}}}}2020-06-19T01:02:05.473Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":440,"time":{"ms" :9}},"total":{"ticks":630,"time":{"ms":11},"value":630},"user":{"ticks":190,"time":{"ms":2}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":9},"info":{"ephemeral_id":"e2f58be5-c311-422f-acf0-540d8acd8950","uptime":{"ms":720107}},"memstats":{"gc_next":5040416,"memory_alloc":3568984,"memory_total":36098000}},"filebeat":{"harvester":{"open_files":0,"running":0}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":1}},"system":{"load":{"1":0,"15":0,"5":0,"norm":{"1":0,"15":0,"5":0}}}}}}2020-06-19T01:02:35.477Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":460,"time":{"ms" :11}},"total":{"ticks":660,"time":{"ms":15},"value":660},"user":{"ticks":200,"time":{"ms":4}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":9},"info":{"ephemeral_id":"e2f58be5-c311-422f-acf0-540d8acd8950","uptime":{"ms":750113}},"memstats":{"gc_next":5040384,"memory_alloc":2528064,"memory_total":36221760,"rss":-139264}},"filebeat":{"harvester":{"open_files":0,"running":0}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":1}},"system":{"load":{"1":0,"15":0,"5":0,"norm":{"1":0,"15":0,"5":0}}}}}}2020-06-19T01:03:05.478Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":460,"time":{"ms" :2}},"total":{"ticks":660,"time":{"ms":7},"value":660},"user":{"ticks":200,"time":{"ms":5}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":9},"info":{"ephemeral_id":"e2f58be5-c311-422f-acf0-540d8acd8950","uptime":{"ms":780111}},"memstats":{"gc_next":5040384,"memory_alloc":2979144,"memory_total":36672840,"rss":262144}},"filebeat":{"harvester":{"open_files":0,"running":0}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":1}},"system":{"load":{"1":0,"15":0,"5":0,"norm":{"1":0,"15":0,"5":0}}}}}}2020-06-19T01:03:35.473Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":470,"time":{"ms" :10}},"total":{"ticks":670,"time":{"ms":11},"value":670},"user":{"ticks":200,"time":{"ms":1}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":9},"info":{"ephemeral_id":"e2f58be5-c311-422f-acf0-540d8acd8950","uptime":{"ms":810107}},"memstats":{"gc_next":5040384,"memory_alloc":3107832,"memory_total":36801528}},"filebeat":{"harvester":{"open_files":0,"running":0}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":1}},"system":{"load":{"1":0,"15":0,"5":0,"norm":{"1":0,"15":0,"5":0}}}}}}2020-06-19T01:03:35.639Z INFO log/harvester.go:255 Harvester started for file: /var/log/syslog 2020-06-19T01:04:05.477Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":480,"time":{"ms" :15}},"total":{"ticks":690,"time":{"ms":18},"value":690},"user":{"ticks":210,"time":{"ms":3}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":10},"info":{"ephemeral_id":"e2f58be5-c311-422f-acf0-540d8acd8950","uptime":{"ms":840109}},"memstats":{"gc_next":5040384,"memory_alloc":3872528,"memory_total":37566224,"rss":249856}},"filebeat":{"events":{"added":2,"done":2},"harvester":{"open_files":1,"running":1,"started":1}},"libbeat":{"config":{"module":{"running":0}},"output":{"events":{"acked":1,"batches":1,"total":1},"read":{"bytes":346},"write":{"bytes":953}},"pipeline":{"clients":1,"events":{"active":0,"filtered":1,"published":1,"total":2},"queue":{"acked":1}}},"registrar":{"states":{"current":1,"update":2},"writes":{"success":2,"total":2}},"system":{"load":{"1":0,"15":0,"5":0,"norm":{"1":0,"15":0,"5":0}}}}}}2020-06-19T01:04:35.473Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":490,"time":{"ms" :7}},"total":{"ticks":700,"time":{"ms":8},"value":700},"user":{"ticks":210,"time":{"ms":1}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":10},"info":{"ephemeral_id":"e2f58be5-c311-422f-acf0-540d8acd8950","uptime":{"ms":870107}},"memstats":{"gc_next":5107536,"memory_alloc":2563640,"memory_total":37784384,"rss":262144}},"filebeat":{"harvester":{"open_files":1,"running":1}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":1}},"system":{"load":{"1":0,"15":0,"5":0,"norm":{"1":0,"15":0,"5":0}}}}}}
3>.在kibana创建索引
三.将数据写入到kafka集群
1>.创建topic
[root@kafka201.yinzhengjie.com ~]# kafka-topics.sh --bootstrap-server 172.200.4.202:9092 --list __consumer_offsets nginx-172-200-5-103 syslog-172-200-5-103 yinzhengjie-kafka [root@kafka201.yinzhengjie.com ~]#
[root@kafka201.yinzhengjie.com ~]# kafka-topics.sh --bootstrap-server 172.200.4.202:9092 --create --partitions 3 --replication-factor 2 --topic filebeat-ubuntu-syslog Created topic filebeat-ubuntu-syslog. [root@kafka201.yinzhengjie.com ~]#
[root@kafka201.yinzhengjie.com ~]# kafka-topics.sh --bootstrap-server 172.200.4.202:9092 --list __consumer_offsets filebeat-ubuntu-syslog nginx-172-200-5-103 syslog-172-200-5-103 yinzhengjie-kafka [root@kafka201.yinzhengjie.com ~]#
2>.配置filebeat
[root@es103.yinzhengjie.com ~]# vim /etc/filebeat/filebeat.yml [root@es103.yinzhengjie.com ~]# [root@es103.yinzhengjie.com ~]# egrep -v "^*#|^$" /etc/filebeat/filebeat.yml filebeat.inputs: - type: log enabled: true paths: - /var/log/syslog fields: host: "172.200.5.103" type: "filebeat-syslog-172-200-5-103" app: "ubuntu-syslog" filebeat.config.modules: path: ${path.config}/modules.d/*.yml reload.enabled: false setup.template.enabled: false setup.template.settings: index.number_of_shards: 3 setup.kibana: output.kafka: hosts: ["kafka201.yinzhengjie.com:9092","kafka202.yinzhengjie.com:9092","kafka203.yinzhengjie.com:9092"] topic: "filebeat-ubuntu-syslog" partition.round_robin: reachable_only: true required_acks: 1 compression: gzip max_message_bytes: 1000000 processors: - add_host_metadata: ~ - add_cloud_metadata: ~ [root@es103.yinzhengjie.com ~]#
[root@es103.yinzhengjie.com ~]# systemctl restart filebeat
[root@es103.yinzhengjie.com ~]# tail -100f /var/log/filebeat/filebeat 2020-06-19T02:06:35.231Z INFO kafka/log.go:53 producer/broker/202 state change to [open] on filebeat-ubuntu-syslog/2 2020-06-19T02:06:35.231Z INFO kafka/log.go:53 producer/leader/filebeat-ubuntu-syslog/2 selected broker 202 2020-06-19T02:06:35.231Z INFO kafka/log.go:53 producer/leader/filebeat-ubuntu-syslog/2 state change to [flushing-1] 2020-06-19T02:06:35.231Z INFO kafka/log.go:53 producer/leader/filebeat-ubuntu-syslog/2 state change to [normal] 2020-06-19T02:06:35.231Z INFO kafka/log.go:53 producer/broker/201 starting up 2020-06-19T02:06:35.231Z INFO kafka/log.go:53 producer/broker/201 state change to [open] on filebeat-ubuntu-syslog/0 2020-06-19T02:06:35.231Z INFO kafka/log.go:53 producer/leader/filebeat-ubuntu-syslog/0 selected broker 201 2020-06-19T02:06:35.231Z INFO kafka/log.go:53 producer/leader/filebeat-ubuntu-syslog/0 state change to [flushing-1] 2020-06-19T02:06:35.231Z INFO kafka/log.go:53 producer/leader/filebeat-ubuntu-syslog/0 state change to [normal] 2020-06-19T02:06:35.232Z INFO kafka/log.go:53 Connected to broker at kafka202.yinzhengjie.com:9092 (registered as #202) 2020-06-19T02:06:35.232Z INFO kafka/log.go:53 Connected to broker at kafka201.yinzhengjie.com:9092 (registered as #201) 2020-06-19T02:06:35.236Z INFO kafka/log.go:53 producer/broker/203 starting up 2020-06-19T02:06:35.236Z INFO kafka/log.go:53 producer/broker/203 state change to [open] on filebeat-ubuntu-syslog/1 2020-06-19T02:06:35.236Z INFO kafka/log.go:53 producer/leader/filebeat-ubuntu-syslog/1 selected broker 203 2020-06-19T02:06:35.236Z INFO kafka/log.go:53 producer/leader/filebeat-ubuntu-syslog/1 state change to [flushing-1] 2020-06-19T02:06:35.236Z INFO kafka/log.go:53 producer/leader/filebeat-ubuntu-syslog/1 state change to [normal] 2020-06-19T02:06:35.237Z INFO kafka/log.go:53 Connected to broker at kafka203.yinzhengjie.com:9092 (registered as #203) 2020-06-19T02:06:54.028Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":230,"time":{"ms" :20}},"total":{"ticks":330,"time":{"ms":40},"value":330},"user":{"ticks":100,"time":{"ms":20}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":11},"info":{"ephemeral_id":"08c37c31-7234-4ef7-b17c-f0381134cd13","uptime":{"ms":810072}},"memstats":{"gc_next":8953232,"memory_alloc":7006160,"memory_total":21025640,"rss":4837376}},"filebeat":{"events":{"added":4,"done":4},"harvester":{"open_files":1,"running":1,"started":1}},"libbeat":{"config":{"module":{"running":0}},"output":{"events":{"acked":3,"batches":1,"total":3}},"outputs":{"kafka":{"bytes_read":981,"bytes_write":3951}},"pipeline":{"clients":1,"events":{"active":0,"filtered":1,"published":3,"total":4},"queue":{"acked":3}}},"registrar":{"states":{"current":1,"update":4},"writes":{"success":2,"total":2}},"system":{"load":{"1":0.01,"15":0,"5":0.01,"norm":{"1":0.005,"15":0,"5":0.005}}}}}}2020-06-19T02:07:24.029Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":230,"time":{"ms" :2}},"total":{"ticks":330,"time":{"ms":8},"value":330},"user":{"ticks":100,"time":{"ms":6}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":11},"info":{"ephemeral_id":"08c37c31-7234-4ef7-b17c-f0381134cd13","uptime":{"ms":840074}},"memstats":{"gc_next":4194304,"memory_alloc":2012136,"memory_total":21412752}},"filebeat":{"harvester":{"open_files":1,"running":1}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":1}},"system":{"load":{"1":0.01,"15":0,"5":0.01,"norm":{"1":0.005,"15":0,"5":0.005}}}}}}2020-06-19T02:07:54.029Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":230,"time":{"ms" :5}},"total":{"ticks":330,"time":{"ms":5},"value":330},"user":{"ticks":100}},"handles":{"limit":{"hard":4096,"soft":1024},"open":11},"info":{"ephemeral_id":"08c37c31-7234-4ef7-b17c-f0381134cd13","uptime":{"ms":870073}},"memstats":{"gc_next":4194304,"memory_alloc":2223992,"memory_total":21624608}},"filebeat":{"harvester":{"open_files":1,"running":1}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":1}},"system":{"load":{"1":0,"15":0,"5":0.01,"norm":{"1":0,"15":0,"5":0.005}}}}}}2020-06-19T02:08:24.032Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":240,"time":{"ms" :9}},"total":{"ticks":340,"time":{"ms":11},"value":340},"user":{"ticks":100,"time":{"ms":2}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":11},"info":{"ephemeral_id":"08c37c31-7234-4ef7-b17c-f0381134cd13","uptime":{"ms":900076}},"memstats":{"gc_next":4194304,"memory_alloc":2601256,"memory_total":22001872}},"filebeat":{"harvester":{"open_files":1,"running":1}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":1}},"system":{"load":{"1":0,"15":0,"5":0,"norm":{"1":0,"15":0,"5":0}}}}}}2020-06-19T02:08:54.028Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":250,"time":{"ms" :5}},"total":{"ticks":360,"time":{"ms":7},"value":360},"user":{"ticks":110,"time":{"ms":2}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":11},"info":{"ephemeral_id":"08c37c31-7234-4ef7-b17c-f0381134cd13","uptime":{"ms":930073}},"memstats":{"gc_next":4194304,"memory_alloc":2812168,"memory_total":22212784}},"filebeat":{"harvester":{"open_files":1,"running":1}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":1}},"system":{"load":{"1":0,"15":0,"5":0,"norm":{"1":0,"15":0,"5":0}}}}}}2020-06-19T02:09:24.028Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":250,"time":{"ms" :6}},"total":{"ticks":360,"time":{"ms":9},"value":360},"user":{"ticks":110,"time":{"ms":3}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":11},"info":{"ephemeral_id":"08c37c31-7234-4ef7-b17c-f0381134cd13","uptime":{"ms":960072}},"memstats":{"gc_next":4194304,"memory_alloc":1879648,"memory_total":22497696,"rss":270336}},"filebeat":{"harvester":{"open_files":1,"running":1}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":1}},"system":{"load":{"1":0.06,"15":0,"5":0.01,"norm":{"1":0.03,"15":0,"5":0.005}}}}}}2020-06-19T02:09:54.029Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":260,"time":{"ms" :4}},"total":{"ticks":370,"time":{"ms":8},"value":370},"user":{"ticks":110,"time":{"ms":4}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":11},"info":{"ephemeral_id":"08c37c31-7234-4ef7-b17c-f0381134cd13","uptime":{"ms":990075}},"memstats":{"gc_next":4194304,"memory_alloc":2185880,"memory_total":22803928}},"filebeat":{"harvester":{"open_files":1,"running":1}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":1}},"system":{"load":{"1":0.03,"15":0,"5":0.01,"norm":{"1":0.015,"15":0,"5":0.005}}}}}}2020-06-19T02:10:24.035Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":260,"time":{"ms" :4}},"total":{"ticks":380,"time":{"ms":14},"value":380},"user":{"ticks":120,"time":{"ms":10}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":11},"info":{"ephemeral_id":"08c37c31-7234-4ef7-b17c-f0381134cd13","uptime":{"ms":1020081}},"memstats":{"gc_next":4194304,"memory_alloc":2480712,"memory_total":23098760}},"filebeat":{"harvester":{"open_files":1,"running":1}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":1}},"system":{"load":{"1":0.02,"15":0,"5":0.01,"norm":{"1":0.01,"15":0,"5":0.005}}}}}}2020-06-19T02:10:54.029Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":260,"time":{"ms" :4}},"total":{"ticks":390,"time":{"ms":9},"value":390},"user":{"ticks":130,"time":{"ms":5}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":11},"info":{"ephemeral_id":"08c37c31-7234-4ef7-b17c-f0381134cd13","uptime":{"ms":1050073}},"memstats":{"gc_next":4194304,"memory_alloc":2769064,"memory_total":23387112}},"filebeat":{"harvester":{"open_files":1,"running":1}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":1}},"system":{"load":{"1":0.01,"15":0,"5":0.01,"norm":{"1":0.005,"15":0,"5":0.005}}}}}}2020-06-19T02:11:24.033Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":280,"time":{"ms" :18}},"total":{"ticks":420,"time":{"ms":30},"value":420},"user":{"ticks":140,"time":{"ms":12}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":11},"info":{"ephemeral_id":"08c37c31-7234-4ef7-b17c-f0381134cd13","uptime":{"ms":1080074}},"memstats":{"gc_next":4194304,"memory_alloc":1888160,"memory_total":23686456}},"filebeat":{"harvester":{"open_files":1,"running":1}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":1}},"system":{"load":{"1":0,"15":0,"5":0,"norm":{"1":0,"15":0,"5":0}}}}}}2020-06-19T02:11:39.150Z INFO log/harvester.go:280 File is inactive: /var/log/syslog. Closing because close_inactive of 5m0s reached. 2020-06-19T02:11:54.028Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":290,"time":{"ms" :10}},"total":{"ticks":430,"time":{"ms":10},"value":430},"user":{"ticks":140}},"handles":{"limit":{"hard":4096,"soft":1024},"open":10},"info":{"ephemeral_id":"08c37c31-7234-4ef7-b17c-f0381134cd13","uptime":{"ms":1110073}},"memstats":{"gc_next":4194304,"memory_alloc":2353896,"memory_total":24152192}},"filebeat":{"events":{"added":1,"done":1},"harvester":{"closed":1,"open_files":0,"running":0}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0,"filtered":1,"total":1}}},"registrar":{"states":{"current":1,"update":1},"writes":{"success":1,"total":1}},"system":{"load":{"1":0,"15":0,"5":0,"norm":{"1":0,"15":0,"5":0}}}}}}2020-06-19T02:12:24.029Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":290},"total":{"t icks":440,"time":{"ms":4},"value":440},"user":{"ticks":150,"time":{"ms":4}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":10},"info":{"ephemeral_id":"08c37c31-7234-4ef7-b17c-f0381134cd13","uptime":{"ms":1140075}},"memstats":{"gc_next":4194304,"memory_alloc":2481992,"memory_total":24280288}},"filebeat":{"harvester":{"open_files":0,"running":0}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":1}},"system":{"load":{"1":0,"15":0,"5":0,"norm":{"1":0,"15":0,"5":0}}}}}}2020-06-19T02:12:54.031Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":290,"time":{"ms" :3}},"total":{"ticks":440,"time":{"ms":8},"value":440},"user":{"ticks":150,"time":{"ms":5}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":10},"info":{"ephemeral_id":"08c37c31-7234-4ef7-b17c-f0381134cd13","uptime":{"ms":1170073}},"memstats":{"gc_next":4194304,"memory_alloc":2770088,"memory_total":24568384}},"filebeat":{"harvester":{"open_files":0,"running":0}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":1}},"system":{"load":{"1":0,"15":0,"5":0,"norm":{"1":0,"15":0,"5":0}}}}}}2020-06-19T02:13:24.032Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":300,"time":{"ms" :8}},"total":{"ticks":460,"time":{"ms":18},"value":460},"user":{"ticks":160,"time":{"ms":10}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":10},"info":{"ephemeral_id":"08c37c31-7234-4ef7-b17c-f0381134cd13","uptime":{"ms":1200074}},"memstats":{"gc_next":4194304,"memory_alloc":1842264,"memory_total":24851568}},"filebeat":{"harvester":{"open_files":0,"running":0}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":1}},"system":{"load":{"1":0,"15":0,"5":0,"norm":{"1":0,"15":0,"5":0}}}}}}2020-06-19T02:13:28.068Z INFO kafka/log.go:53 client/metadata fetching metadata for all topics from broker kafka203.yinzhengjie.com:9092 2020-06-19T02:13:54.028Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":310,"time":{"ms" :6}},"total":{"ticks":470,"time":{"ms":9},"value":470},"user":{"ticks":160,"time":{"ms":3}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":10},"info":{"ephemeral_id":"08c37c31-7234-4ef7-b17c-f0381134cd13","uptime":{"ms":1230073}},"memstats":{"gc_next":4194304,"memory_alloc":2170560,"memory_total":25179864,"rss":-4644864}},"filebeat":{"harvester":{"open_files":0,"running":0}},"libbeat":{"config":{"module":{"running":0}},"outputs":{"kafka":{"bytes_read":3831,"bytes_write":23}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":1}},"system":{"load":{"1":0,"15":0,"5":0,"norm":{"1":0,"15":0,"5":0}}}}}}2020-06-19T02:14:24.028Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":310},"total":{"t icks":480,"time":{"ms":4},"value":480},"user":{"ticks":170,"time":{"ms":4}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":10},"info":{"ephemeral_id":"08c37c31-7234-4ef7-b17c-f0381134cd13","uptime":{"ms":1260073}},"memstats":{"gc_next":4194304,"memory_alloc":2462624,"memory_total":25471928}},"filebeat":{"harvester":{"open_files":0,"running":0}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":1}},"system":{"load":{"1":0,"15":0,"5":0,"norm":{"1":0,"15":0,"5":0}}}}}}2020-06-19T02:14:54.029Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":310,"time":{"ms" :1}},"total":{"ticks":480,"time":{"ms":6},"value":480},"user":{"ticks":170,"time":{"ms":5}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":10},"info":{"ephemeral_id":"08c37c31-7234-4ef7-b17c-f0381134cd13","uptime":{"ms":1290074}},"memstats":{"gc_next":4194304,"memory_alloc":2833920,"memory_total":25843224}},"filebeat":{"harvester":{"open_files":0,"running":0}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":1}},"system":{"load":{"1":0,"15":0,"5":0,"norm":{"1":0,"15":0,"5":0}}}}}}2020-06-19T02:15:24.028Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":310,"time":{"ms" :3}},"total":{"ticks":490,"time":{"ms":15},"value":490},"user":{"ticks":180,"time":{"ms":12}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":10},"info":{"ephemeral_id":"08c37c31-7234-4ef7-b17c-f0381134cd13","uptime":{"ms":1320073}},"memstats":{"gc_next":4194304,"memory_alloc":1851272,"memory_total":26054472}},"filebeat":{"harvester":{"open_files":0,"running":0}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":1}},"system":{"load":{"1":0,"15":0,"5":0,"norm":{"1":0,"15":0,"5":0}}}}}}2020-06-19T02:15:54.028Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":320,"time":{"ms" :5}},"total":{"ticks":500,"time":{"ms":5},"value":500},"user":{"ticks":180}},"handles":{"limit":{"hard":4096,"soft":1024},"open":10},"info":{"ephemeral_id":"08c37c31-7234-4ef7-b17c-f0381134cd13","uptime":{"ms":1350073}},"memstats":{"gc_next":4194304,"memory_alloc":2309232,"memory_total":26512432}},"filebeat":{"harvester":{"open_files":0,"running":0}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":1}},"system":{"load":{"1":0,"15":0,"5":0,"norm":{"1":0,"15":0,"5":0}}}}}}2020-06-19T02:16:24.029Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":320,"time":{"ms" :4}},"total":{"ticks":510,"time":{"ms":6},"value":510},"user":{"ticks":190,"time":{"ms":2}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":10},"info":{"ephemeral_id":"08c37c31-7234-4ef7-b17c-f0381134cd13","uptime":{"ms":1380073}},"memstats":{"gc_next":4194304,"memory_alloc":2437472,"memory_total":26640672}},"filebeat":{"harvester":{"open_files":0,"running":0}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":1}},"system":{"load":{"1":0,"15":0,"5":0,"norm":{"1":0,"15":0,"5":0}}}}}}2020-06-19T02:16:54.029Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":330,"time":{"ms" :4}},"total":{"ticks":520,"time":{"ms":4},"value":520},"user":{"ticks":190}},"handles":{"limit":{"hard":4096,"soft":1024},"open":10},"info":{"ephemeral_id":"08c37c31-7234-4ef7-b17c-f0381134cd13","uptime":{"ms":1410074}},"memstats":{"gc_next":4194304,"memory_alloc":2725776,"memory_total":26928976}},"filebeat":{"harvester":{"open_files":0,"running":0}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":1}},"system":{"load":{"1":0,"15":0,"5":0,"norm":{"1":0,"15":0,"5":0}}}}}}2020-06-19T02:17:04.177Z INFO log/harvester.go:255 Harvester started for file: /var/log/syslog 2020-06-19T02:17:05.181Z INFO kafka/log.go:53 producer/broker/201 state change to [closing] because EOF 2020-06-19T02:17:05.182Z INFO kafka/log.go:53 Closed connection to broker kafka201.yinzhengjie.com:9092 2020-06-19T02:17:05.182Z INFO kafka/log.go:53 producer/leader/filebeat-ubuntu-syslog/0 state change to [retrying-1] 2020-06-19T02:17:05.182Z INFO kafka/log.go:53 producer/leader/filebeat-ubuntu-syslog/0 abandoning broker 201 2020-06-19T02:17:05.182Z INFO kafka/log.go:53 producer/broker/201 shut down 2020-06-19T02:17:05.282Z INFO kafka/log.go:53 client/metadata fetching metadata for [filebeat-ubuntu-syslog] from broker kafka203.yinzhengjie.com:9092 2020-06-19T02:17:05.285Z INFO kafka/log.go:53 producer/broker/201 starting up 2020-06-19T02:17:05.285Z INFO kafka/log.go:53 producer/broker/201 state change to [open] on filebeat-ubuntu-syslog/0 2020-06-19T02:17:05.285Z INFO kafka/log.go:53 producer/leader/filebeat-ubuntu-syslog/0 selected broker 201 2020-06-19T02:17:05.285Z INFO kafka/log.go:53 producer/leader/filebeat-ubuntu-syslog/0 state change to [flushing-1] 2020-06-19T02:17:05.285Z INFO kafka/log.go:53 producer/leader/filebeat-ubuntu-syslog/0 state change to [normal] 2020-06-19T02:17:05.286Z INFO kafka/log.go:53 Connected to broker at kafka201.yinzhengjie.com:9092 (registered as #201) 2020-06-19T02:17:24.028Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":350,"time":{"ms" :21}},"total":{"ticks":540,"time":{"ms":25},"value":540},"user":{"ticks":190,"time":{"ms":4}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":11},"info":{"ephemeral_id":"08c37c31-7234-4ef7-b17c-f0381134cd13","uptime":{"ms":1440073}},"memstats":{"gc_next":5652752,"memory_alloc":2844720,"memory_total":29211296,"rss":2879488}},"filebeat":{"events":{"added":2,"done":2},"harvester":{"open_files":1,"running":1,"started":1}},"libbeat":{"config":{"module":{"running":0}},"output":{"events":{"acked":1,"batches":1,"total":1}},"outputs":{"kafka":{"bytes_read":327,"bytes_write":1317}},"pipeline":{"clients":1,"events":{"active":0,"filtered":1,"published":1,"total":2},"queue":{"acked":1}}},"registrar":{"states":{"current":1,"update":2},"writes":{"success":2,"total":2}},"system":{"load":{"1":0,"15":0,"5":0,"norm":{"1":0,"15":0,"5":0}}}}}}2020-06-19T02:17:54.028Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":350,"time":{"ms" :5}},"total":{"ticks":540,"time":{"ms":5},"value":540},"user":{"ticks":190}},"handles":{"limit":{"hard":4096,"soft":1024},"open":11},"info":{"ephemeral_id":"08c37c31-7234-4ef7-b17c-f0381134cd13","uptime":{"ms":1470073}},"memstats":{"gc_next":5652752,"memory_alloc":3305488,"memory_total":29672064}},"filebeat":{"harvester":{"open_files":1,"running":1}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":1}},"system":{"load":{"1":0,"15":0,"5":0,"norm":{"1":0,"15":0,"5":0}}}}}}2020-06-19T02:18:24.028Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":360,"time":{"ms" :7}},"total":{"ticks":550,"time":{"ms":8},"value":550},"user":{"ticks":190,"time":{"ms":1}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":11},"info":{"ephemeral_id":"08c37c31-7234-4ef7-b17c-f0381134cd13","uptime":{"ms":1500073}},"memstats":{"gc_next":5652752,"memory_alloc":3444016,"memory_total":29810592}},"filebeat":{"harvester":{"open_files":1,"running":1}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":1}},"system":{"load":{"1":0,"15":0,"5":0,"norm":{"1":0,"15":0,"5":0}}}}}}2020-06-19T02:18:54.029Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":360,"time":{"ms" :4}},"total":{"ticks":550,"time":{"ms":5},"value":550},"user":{"ticks":190,"time":{"ms":1}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":11},"info":{"ephemeral_id":"08c37c31-7234-4ef7-b17c-f0381134cd13","uptime":{"ms":1530074}},"memstats":{"gc_next":5652752,"memory_alloc":3905888,"memory_total":30272464,"rss":-1269760}},"filebeat":{"harvester":{"open_files":1,"running":1}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":1}},"system":{"load":{"1":0,"15":0,"5":0,"norm":{"1":0,"15":0,"5":0}}}}}}2020-06-19T02:19:24.029Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":370,"time":{"ms" :9}},"total":{"ticks":560,"time":{"ms":10},"value":560},"user":{"ticks":190,"time":{"ms":1}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":11},"info":{"ephemeral_id":"08c37c31-7234-4ef7-b17c-f0381134cd13","uptime":{"ms":1560073}},"memstats":{"gc_next":4194304,"memory_alloc":1960296,"memory_total":30471528}},"filebeat":{"harvester":{"open_files":1,"running":1}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":1}},"system":{"load":{"1":0,"15":0,"5":0,"norm":{"1":0,"15":0,"5":0}}}}}}2020-06-19T02:19:54.028Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":380,"time":{"ms" :7}},"total":{"ticks":580,"time":{"ms":9},"value":580},"user":{"ticks":200,"time":{"ms":2}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":11},"info":{"ephemeral_id":"08c37c31-7234-4ef7-b17c-f0381134cd13","uptime":{"ms":1590074}},"memstats":{"gc_next":4194304,"memory_alloc":2179512,"memory_total":30690744}},"filebeat":{"harvester":{"open_files":1,"running":1}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":1}},"system":{"load":{"1":0,"15":0,"5":0,"norm":{"1":0,"15":0,"5":0}}}}}}2020-06-19T02:20:24.029Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":390,"time":{"ms" :6}},"total":{"ticks":590,"time":{"ms":6},"value":590},"user":{"ticks":200}},"handles":{"limit":{"hard":4096,"soft":1024},"open":11},"info":{"ephemeral_id":"08c37c31-7234-4ef7-b17c-f0381134cd13","uptime":{"ms":1620073}},"memstats":{"gc_next":4194304,"memory_alloc":2480664,"memory_total":30991896}},"filebeat":{"harvester":{"open_files":1,"running":1}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":1}},"system":{"load":{"1":0.07,"15":0,"5":0.02,"norm":{"1":0.035,"15":0,"5":0.01}}}}}}2020-06-19T02:20:54.028Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":390,"time":{"ms" :5}},"total":{"ticks":590,"time":{"ms":5},"value":590},"user":{"ticks":200}},"handles":{"limit":{"hard":4096,"soft":1024},"open":11},"info":{"ephemeral_id":"08c37c31-7234-4ef7-b17c-f0381134cd13","uptime":{"ms":1650074}},"memstats":{"gc_next":4194304,"memory_alloc":2772088,"memory_total":31283320}},"filebeat":{"harvester":{"open_files":1,"running":1}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":1}},"system":{"load":{"1":0.04,"15":0,"5":0.01,"norm":{"1":0.02,"15":0,"5":0.005}}}}}}2020-06-19T02:21:24.029Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":390,"time":{"ms" :2}},"total":{"ticks":600,"time":{"ms":17},"value":600},"user":{"ticks":210,"time":{"ms":15}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":11},"info":{"ephemeral_id":"08c37c31-7234-4ef7-b17c-f0381134cd13","uptime":{"ms":1680074}},"memstats":{"gc_next":4194304,"memory_alloc":2042024,"memory_total":31733344}},"filebeat":{"harvester":{"open_files":1,"running":1}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":1}},"system":{"load":{"1":0.02,"15":0,"5":0.01,"norm":{"1":0.01,"15":0,"5":0.005}}}}}}2020-06-19T02:21:54.028Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":400,"time":{"ms" :7}},"total":{"ticks":610,"time":{"ms":7},"value":610},"user":{"ticks":210}},"handles":{"limit":{"hard":4096,"soft":1024},"open":11},"info":{"ephemeral_id":"08c37c31-7234-4ef7-b17c-f0381134cd13","uptime":{"ms":1710073}},"memstats":{"gc_next":4194304,"memory_alloc":2183912,"memory_total":31875232}},"filebeat":{"harvester":{"open_files":1,"running":1}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":1}},"system":{"load":{"1":0.01,"15":0,"5":0.01,"norm":{"1":0.005,"15":0,"5":0.005}}}}}}2020-06-19T02:22:09.210Z INFO log/harvester.go:280 File is inactive: /var/log/syslog. Closing because close_inactive of 5m0s reached. 2020-06-19T02:22:24.028Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":410,"time":{"ms" :7}},"total":{"ticks":620,"time":{"ms":7},"value":620},"user":{"ticks":210}},"handles":{"limit":{"hard":4096,"soft":1024},"open":10},"info":{"ephemeral_id":"08c37c31-7234-4ef7-b17c-f0381134cd13","uptime":{"ms":1740073}},"memstats":{"gc_next":4194304,"memory_alloc":2561832,"memory_total":32253152}},"filebeat":{"events":{"added":1,"done":1},"harvester":{"closed":1,"open_files":0,"running":0}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0,"filtered":1,"total":1}}},"registrar":{"states":{"current":1,"update":1},"writes":{"success":1,"total":1}},"system":{"load":{"1":0.01,"15":0,"5":0,"norm":{"1":0.005,"15":0,"5":0}}}}}}2020-06-19T02:22:54.028Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":410,"time":{"ms" :4}},"total":{"ticks":620,"time":{"ms":4},"value":620},"user":{"ticks":210}},"handles":{"limit":{"hard":4096,"soft":1024},"open":10},"info":{"ephemeral_id":"08c37c31-7234-4ef7-b17c-f0381134cd13","uptime":{"ms":1770073}},"memstats":{"gc_next":4194304,"memory_alloc":2771384,"memory_total":32462704}},"filebeat":{"harvester":{"open_files":0,"running":0}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":1}},"system":{"load":{"1":0,"15":0,"5":0,"norm":{"1":0,"15":0,"5":0}}}}}}2020-06-19T02:23:24.034Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":410,"time":{"ms" :2}},"total":{"ticks":630,"time":{"ms":12},"value":630},"user":{"ticks":220,"time":{"ms":10}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":10},"info":{"ephemeral_id":"08c37c31-7234-4ef7-b17c-f0381134cd13","uptime":{"ms":1800078}},"memstats":{"gc_next":4194304,"memory_alloc":2013696,"memory_total":32914696}},"filebeat":{"harvester":{"open_files":0,"running":0}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":1}},"system":{"load":{"1":0,"15":0,"5":0,"norm":{"1":0,"15":0,"5":0}}}}}}2020-06-19T02:23:28.069Z INFO kafka/log.go:53 client/metadata fetching metadata for all topics from broker kafka203.yinzhengjie.com:9092 2020-06-19T02:23:54.028Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":430,"time":{"ms" :13}},"total":{"ticks":650,"time":{"ms":14},"value":650},"user":{"ticks":220,"time":{"ms":1}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":10},"info":{"ephemeral_id":"08c37c31-7234-4ef7-b17c-f0381134cd13","uptime":{"ms":1830073}},"memstats":{"gc_next":4194304,"memory_alloc":2170032,"memory_total":33071032}},"filebeat":{"harvester":{"open_files":0,"running":0}},"libbeat":{"config":{"module":{"running":0}},"outputs":{"kafka":{"bytes_read":3831,"bytes_write":23}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":1}},"system":{"load":{"1":0,"15":0,"5":0,"norm":{"1":0,"15":0,"5":0}}}}}}2020-06-19T02:24:24.028Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":430,"time":{"ms" :4}},"total":{"ticks":650,"time":{"ms":4},"value":650},"user":{"ticks":220}},"handles":{"limit":{"hard":4096,"soft":1024},"open":10},"info":{"ephemeral_id":"08c37c31-7234-4ef7-b17c-f0381134cd13","uptime":{"ms":1860073}},"memstats":{"gc_next":4194304,"memory_alloc":2460960,"memory_total":33361960}},"filebeat":{"harvester":{"open_files":0,"running":0}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":1,"events":{"active":0}}},"registrar":{"states":{"current":1}},"system":{"load":{"1":0,"15":0,"5":0,"norm":{"1":0,"15":0,"5":0}}}}}}
3>.测试消费kafka的数据
[root@kafka201.yinzhengjie.com ~]# kafka-console-consumer.sh --bootstrap-server 172.200.4.202:9092 --topic filebeat-ubuntu-syslog {"@timestamp":"2020-06-19T01:53:24.039Z","@metadata":{"beat":"filebeat","type":"doc","version":"6.8.9","topic":"filebeat-ubuntu-syslog"},"offset":267672,"input":{"type":"log"},"host":{"os": {"codename":"bionic","platform":"ubuntu","version":"18.04.3 LTS (Bionic Beaver)","family":"debian","name":"Ubuntu"},"id":"8e0a6eabcd8c40f19da6b7f090c4cd2a","name":"es103.yinzhengjie.com","containerized":false,"architecture":"x86_64"},"log":{"file":{"path":"/var/log/syslog"}},"source":"/var/log/syslog","fields":{"host":"172.200.5.103","type":"filebeat-syslog-172-200-5-103","app":"ubuntu-syslog"},"beat":{"version":"6.8.9","name":"es103.yinzhengjie.com","hostname":"es103.yinzhengjie.com"},"message":"Jun 19 01:53:23 es103 systemd[1]: Stopping Filebeat sends log files to Logstash or directly to Elasticsearch....","prospector":{"type":"log"}}{"@timestamp":"2020-06-19T01:53:27.023Z","@metadata":{"beat":"filebeat","type":"doc","version":"6.8.9","topic":"filebeat-ubuntu-syslog"},"source":"/var/log/syslog","log":{"file":{"path":"/v ar/log/syslog"}},"beat":{"name":"es103.yinzhengjie.com","hostname":"es103.yinzhengjie.com","version":"6.8.9"},"offset":267785,"message":"Jun 19 01:53:23 es103 systemd[1]: Stopped Filebeat sends log files to Logstash or directly to Elasticsearch..","prospector":{"type":"log"},"input":{"type":"log"},"fields":{"type":"filebeat-syslog-172-200-5-103","app":"ubuntu-syslog","host":"172.200.5.103"},"host":{"id":"8e0a6eabcd8c40f19da6b7f090c4cd2a","containerized":false,"name":"es103.yinzhengjie.com","architecture":"x86_64","os":{"codename":"bionic","platform":"ubuntu","version":"18.04.3 LTS (Bionic Beaver)","family":"debian","name":"Ubuntu"}}}{"@timestamp":"2020-06-19T01:53:27.023Z","@metadata":{"beat":"filebeat","type":"doc","version":"6.8.9","topic":"filebeat-ubuntu-syslog"},"beat":{"name":"es103.yinzhengjie.com","hostname":"e s103.yinzhengjie.com","version":"6.8.9"},"source":"/var/log/syslog","log":{"file":{"path":"/var/log/syslog"}},"fields":{"app":"ubuntu-syslog","host":"172.200.5.103","type":"filebeat-syslog-172-200-5-103"},"prospector":{"type":"log"},"offset":267895,"message":"Jun 19 01:53:23 es103 systemd[1]: Started Filebeat sends log files to Logstash or directly to Elasticsearch..","input":{"type":"log"},"host":{"architecture":"x86_64","name":"es103.yinzhengjie.com","os":{"platform":"ubuntu","version":"18.04.3 LTS (Bionic Beaver)","family":"debian","name":"Ubuntu","codename":"bionic"},"id":"8e0a6eabcd8c40f19da6b7f090c4cd2a","containerized":false}}{"@timestamp":"2020-06-19T02:06:34.119Z","@metadata":{"beat":"filebeat","type":"doc","version":"6.8.9","topic":"filebeat-ubuntu-syslog"},"log":{"file":{"path":"/var/log/syslog"}},"message": "Jun 19 02:06:31 es103 snapd[1250]: autorefresh.go:397: auto-refresh: all snaps are up-to-date","fields":{"type":"filebeat-syslog-172-200-5-103","app":"ubuntu-syslog","host":"172.200.5.103"},"beat":{"name":"es103.yinzhengjie.com","hostname":"es103.yinzhengjie.com","version":"6.8.9"},"source":"/var/log/syslog","offset":268252,"prospector":{"type":"log"},"input":{"type":"log"},"host":{"containerized":false,"architecture":"x86_64","os":{"name":"Ubuntu","codename":"bionic","platform":"ubuntu","version":"18.04.3 LTS (Bionic Beaver)","family":"debian"},"name":"es103.yinzhengjie.com","id":"8e0a6eabcd8c40f19da6b7f090c4cd2a"}}{"@timestamp":"2020-06-19T02:06:34.117Z","@metadata":{"beat":"filebeat","type":"doc","version":"6.8.9","topic":"filebeat-ubuntu-syslog"},"source":"/var/log/syslog","input":{"type":"log"},"p rospector":{"type":"log"},"host":{"name":"es103.yinzhengjie.com","architecture":"x86_64","os":{"family":"debian","name":"Ubuntu","codename":"bionic","platform":"ubuntu","version":"18.04.3 LTS (Bionic Beaver)"},"id":"8e0a6eabcd8c40f19da6b7f090c4cd2a","containerized":false},"message":"Jun 19 02:06:27 es103 systemd-resolved[1103]: Grace period over, resuming full feature set (UDP+EDNS0) for DNS server 172.200.7.254.","log":{"file":{"path":"/var/log/syslog"}},"fields":{"host":"172.200.5.103","type":"filebeat-syslog-172-200-5-103","app":"ubuntu-syslog"},"beat":{"name":"es103.yinzhengjie.com","hostname":"es103.yinzhengjie.com","version":"6.8.9"},"offset":268005}{"@timestamp":"2020-06-19T02:06:34.119Z","@metadata":{"beat":"filebeat","type":"doc","version":"6.8.9","topic":"filebeat-ubuntu-syslog"},"input":{"type":"log"},"beat":{"version":"6.8.9","na me":"es103.yinzhengjie.com","hostname":"es103.yinzhengjie.com"},"message":"Jun 19 02:06:31 es103 snapd[1250]: storehelpers.go:438: cannot refresh snap "core": snap has no updates available","source":"/var/log/syslog","offset":268138,"log":{"file":{"path":"/var/log/syslog"}},"fields":{"type":"filebeat-syslog-172-200-5-103","app":"ubuntu-syslog","host":"172.200.5.103"},"prospector":{"type":"log"},"host":{"architecture":"x86_64","os":{"codename":"bionic","platform":"ubuntu","version":"18.04.3 LTS (Bionic Beaver)","family":"debian","name":"Ubuntu"},"id":"8e0a6eabcd8c40f19da6b7f090c4cd2a","containerized":false,"name":"es103.yinzhengjie.com"}}{"@timestamp":"2020-06-19T02:17:04.177Z","@metadata":{"beat":"filebeat","type":"doc","version":"6.8.9","topic":"filebeat-ubuntu-syslog"},"offset":268346,"log":{"file":{"path":"/var/log/sysl og"}},"source":"/var/log/syslog","input":{"type":"log"},"fields":{"type":"filebeat-syslog-172-200-5-103","app":"ubuntu-syslog","host":"172.200.5.103"},"prospector":{"type":"log"},"beat":{"name":"es103.yinzhengjie.com","hostname":"es103.yinzhengjie.com","version":"6.8.9"},"message":"Jun 19 02:17:01 es103 CRON[3407]: (root) CMD ( cd / u0026u0026 run-parts --report /etc/cron.hourly)","host":{"architecture":"x86_64","os":{"version":"18.04.3 LTS (Bionic Beaver)","family":"debian","name":"Ubuntu","codename":"bionic","platform":"ubuntu"},"id":"8e0a6eabcd8c40f19da6b7f090c4cd2a","containerized":false,"name":"es103.yinzhengjie.com"}}
4>.使用logstash消费写入kafka的数据
[root@es103.yinzhengjie.com ~]# vim /etc/logstash/conf.d/kafka-to-elasticsearch.conf [root@es103.yinzhengjie.com ~]# [root@es103.yinzhengjie.com ~]# cat /etc/logstash/conf.d/kafka-to-elasticsearch.conf input { kafka { topics => ["filebeat-ubuntu-syslog"] bootstrap_servers => "kafka201.yinzhengjie.com:9092,kafka202.yinzhengjie.com:9092,kafka203.yinzhengjie.com:9092" codec => "json" group_id => "yinzhengjie-logstash" decorate_events => true consumer_threads => 3 } } output{ if [fields][app] == "ubuntu-syslog" { elasticsearch { hosts => ["http://es101.yinzhengjie.com:9200","http://es102.yinzhengjie.com:9200"] index => "filebeat-syslog-172.200.5.103-%{+yyyy+MM+dd}" } } } [root@es103.yinzhengjie.com ~]#
[root@es103.yinzhengjie.com ~]# systemctl restart logstash.service
[root@es103.yinzhengjie.com ~]# tail -100f /var/log/logstash/logstash-plain.log [2020-06-19T03:16:12,899][INFO ][org.apache.kafka.common.utils.AppInfoParser] Kafka version : 2.1.0 [2020-06-19T03:16:12,899][INFO ][org.apache.kafka.common.utils.AppInfoParser] Kafka commitId : eec43959745f444f [2020-06-19T03:16:12,903][INFO ][org.apache.kafka.clients.consumer.ConsumerConfig] ConsumerConfig values: auto.commit.interval.ms = 5000 auto.offset.reset = latest bootstrap.servers = [kafka201.yinzhengjie.com:9092, kafka202.yinzhengjie.com:9092, kafka203.yinzhengjie.com:9092] check.crcs = true client.dns.lookup = default client.id = logstash-2 connections.max.idle.ms = 540000 default.api.timeout.ms = 60000 enable.auto.commit = true exclude.internal.topics = true fetch.max.bytes = 52428800 fetch.max.wait.ms = 500 fetch.min.bytes = 1 group.id = yinzhengjie-logstash heartbeat.interval.ms = 3000 interceptor.classes = [] internal.leave.group.on.close = true isolation.level = read_uncommitted key.deserializer = class org.apache.kafka.common.serialization.StringDeserializer max.partition.fetch.bytes = 1048576 max.poll.interval.ms = 300000 max.poll.records = 500 metadata.max.age.ms = 300000 metric.reporters = [] metrics.num.samples = 2 metrics.recording.level = INFO metrics.sample.window.ms = 30000 partition.assignment.strategy = [class org.apache.kafka.clients.consumer.RangeAssignor] receive.buffer.bytes = 65536 reconnect.backoff.max.ms = 1000 reconnect.backoff.ms = 50 request.timeout.ms = 30000 retry.backoff.ms = 100 sasl.client.callback.handler.class = null sasl.jaas.config = null sasl.kerberos.kinit.cmd = /usr/bin/kinit sasl.kerberos.min.time.before.relogin = 60000 sasl.kerberos.service.name = null sasl.kerberos.ticket.renew.jitter = 0.05 sasl.kerberos.ticket.renew.window.factor = 0.8 sasl.login.callback.handler.class = null sasl.login.class = null sasl.login.refresh.buffer.seconds = 300 sasl.login.refresh.min.period.seconds = 60 sasl.login.refresh.window.factor = 0.8 sasl.login.refresh.window.jitter = 0.05 sasl.mechanism = GSSAPI security.protocol = PLAINTEXT send.buffer.bytes = 131072 session.timeout.ms = 10000 ssl.cipher.suites = null ssl.enabled.protocols = [TLSv1.2, TLSv1.1, TLSv1] ssl.endpoint.identification.algorithm = https ssl.key.password = null ssl.keymanager.algorithm = SunX509 ssl.keystore.location = null ssl.keystore.password = null ssl.keystore.type = JKS ssl.protocol = TLS ssl.provider = null ssl.secure.random.implementation = null ssl.trustmanager.algorithm = PKIX ssl.truststore.location = null ssl.truststore.password = null ssl.truststore.type = JKS value.deserializer = class org.apache.kafka.common.serialization.StringDeserializer [2020-06-19T03:16:12,913][INFO ][org.apache.kafka.common.utils.AppInfoParser] Kafka version : 2.1.0 [2020-06-19T03:16:12,914][INFO ][org.apache.kafka.common.utils.AppInfoParser] Kafka commitId : eec43959745f444f [2020-06-19T03:16:15,761][INFO ][org.apache.kafka.clients.Metadata] Cluster ID: aDBW0CNdQeC8lxYFd77sxg [2020-06-19T03:16:15,761][INFO ][org.apache.kafka.clients.Metadata] Cluster ID: aDBW0CNdQeC8lxYFd77sxg [2020-06-19T03:16:15,761][INFO ][org.apache.kafka.clients.Metadata] Cluster ID: aDBW0CNdQeC8lxYFd77sxg [2020-06-19T03:16:16,042][INFO ][org.apache.kafka.clients.consumer.internals.AbstractCoordinator] [Consumer clientId=logstash-1, groupId=yinzhengjie-logstash] Discovered group coordinator k afka203.yinzhengjie.com:9092 (id: 2147483444 rack: null)[2020-06-19T03:16:16,042][INFO ][org.apache.kafka.clients.consumer.internals.AbstractCoordinator] [Consumer clientId=logstash-0, groupId=yinzhengjie-logstash] Discovered group coordinator k afka203.yinzhengjie.com:9092 (id: 2147483444 rack: null)[2020-06-19T03:16:16,042][INFO ][org.apache.kafka.clients.consumer.internals.AbstractCoordinator] [Consumer clientId=logstash-2, groupId=yinzhengjie-logstash] Discovered group coordinator k afka203.yinzhengjie.com:9092 (id: 2147483444 rack: null)[2020-06-19T03:16:16,172][INFO ][org.apache.kafka.clients.consumer.internals.ConsumerCoordinator] [Consumer clientId=logstash-0, groupId=yinzhengjie-logstash] Revoking previously assigned p artitions [][2020-06-19T03:16:16,173][INFO ][org.apache.kafka.clients.consumer.internals.AbstractCoordinator] [Consumer clientId=logstash-0, groupId=yinzhengjie-logstash] (Re-)joining group [2020-06-19T03:16:16,173][INFO ][org.apache.kafka.clients.consumer.internals.ConsumerCoordinator] [Consumer clientId=logstash-2, groupId=yinzhengjie-logstash] Revoking previously assigned p artitions [][2020-06-19T03:16:16,174][INFO ][org.apache.kafka.clients.consumer.internals.AbstractCoordinator] [Consumer clientId=logstash-2, groupId=yinzhengjie-logstash] (Re-)joining group [2020-06-19T03:16:16,175][INFO ][org.apache.kafka.clients.consumer.internals.ConsumerCoordinator] [Consumer clientId=logstash-1, groupId=yinzhengjie-logstash] Revoking previously assigned p artitions [][2020-06-19T03:16:16,208][INFO ][org.apache.kafka.clients.consumer.internals.AbstractCoordinator] [Consumer clientId=logstash-1, groupId=yinzhengjie-logstash] (Re-)joining group [2020-06-19T03:16:19,461][INFO ][org.apache.kafka.clients.consumer.internals.AbstractCoordinator] [Consumer clientId=logstash-2, groupId=yinzhengjie-logstash] Successfully joined group with generation 14[2020-06-19T03:16:19,539][INFO ][org.apache.kafka.clients.consumer.internals.AbstractCoordinator] [Consumer clientId=logstash-1, groupId=yinzhengjie-logstash] Successfully joined group with generation 14[2020-06-19T03:16:19,461][INFO ][org.apache.kafka.clients.consumer.internals.AbstractCoordinator] [Consumer clientId=logstash-0, groupId=yinzhengjie-logstash] Successfully joined group with generation 14[2020-06-19T03:16:19,626][INFO ][org.apache.kafka.clients.consumer.internals.ConsumerCoordinator] [Consumer clientId=logstash-1, groupId=yinzhengjie-logstash] Setting newly assigned partiti ons [filebeat-ubuntu-syslog-1][2020-06-19T03:16:19,627][INFO ][org.apache.kafka.clients.consumer.internals.ConsumerCoordinator] [Consumer clientId=logstash-2, groupId=yinzhengjie-logstash] Setting newly assigned partiti ons [filebeat-ubuntu-syslog-2][2020-06-19T03:16:19,627][INFO ][org.apache.kafka.clients.consumer.internals.ConsumerCoordinator] [Consumer clientId=logstash-0, groupId=yinzhengjie-logstash] Setting newly assigned partiti ons [filebeat-ubuntu-syslog-0][2020-06-19T03:16:19,892][INFO ][org.apache.kafka.clients.consumer.internals.Fetcher] [Consumer clientId=logstash-0, groupId=yinzhengjie-logstash] Resetting offset for partition filebeat-ub untu-syslog-0 to offset 125.[2020-06-19T03:16:19,893][INFO ][org.apache.kafka.clients.consumer.internals.Fetcher] [Consumer clientId=logstash-2, groupId=yinzhengjie-logstash] Resetting offset for partition filebeat-ub untu-syslog-2 to offset 124.[2020-06-19T03:16:19,892][INFO ][org.apache.kafka.clients.consumer.internals.Fetcher] [Consumer clientId=logstash-1, groupId=yinzhengjie-logstash] Resetting offset for partition filebeat-ub untu-syslog-1 to offset 125.[2020-06-19T03:16:21,170][WARN ][filewatch.tailmode.handlers.create] failed to open /var/log/syslog: #<Errno::EACCES: Permission denied - /var/log/syslog>, ["org/jruby/RubyIO.java:1236:in ` sysopen'", "org/jruby/RubyFile.java:367:in `initialize'", "org/jruby/RubyIO.java:1155:in `open'"][2020-06-19T03:16:26,580][INFO ][logstash.agent ] Successfully started Logstash API endpoint {:port=>9600} [2020-06-19T03:20:58,634][INFO ][org.apache.kafka.clients.Metadata] Cluster ID: aDBW0CNdQeC8lxYFd77sxg [2020-06-19T03:20:59,346][INFO ][org.apache.kafka.clients.Metadata] Cluster ID: aDBW0CNdQeC8lxYFd77sxg [2020-06-19T03:21:23,050][WARN ][filewatch.tailmode.handlers.create] failed to open /var/log/syslog: #<Errno::EACCES: Permission denied - /var/log/syslog>, ["org/jruby/RubyIO.java:1236:in ` sysopen'", "org/jruby/RubyFile.java:367:in `initialize'", "org/jruby/RubyIO.java:1155:in `open'"][2020-06-19T03:26:24,723][WARN ][filewatch.tailmode.handlers.create] failed to open /var/log/syslog: #<Errno::EACCES: Permission denied - /var/log/syslog>, ["org/jruby/RubyIO.java:1236:in ` sysopen'", "org/jruby/RubyFile.java:367:in `initialize'", "org/jruby/RubyIO.java:1155:in `open'"]
5>.在Kibana建立索引
四.使用filebeat收集多个日志案例
1>.使用filebeat收集nginx日志和系统日志案例
[root@es103.yinzhengjie.com ~]# vim /etc/filebeat/filebeat.yml [root@es103.yinzhengjie.com ~]# [root@es103.yinzhengjie.com ~]# egrep -v "^*#|^$" /etc/filebeat/filebeat.yml filebeat.inputs: - type: log enabled: true paths: - /var/log/syslog fields: host: "172.200.5.103" type: "filebeat-syslog-172-200-5-103" app: "ubuntu-syslog" - type: log enable: true paths: - /var/log/nginx/access.log fields: host: "172.200.5.103" type: "filebeat-nginx-accesslog-172-200-5-103" app: "nginx" filebeat.config.modules: path: ${path.config}/modules.d/*.yml reload.enabled: false setup.template.enabled: false setup.template.settings: index.number_of_shards: 3 setup.kibana: output.kafka: hosts: ["kafka201.yinzhengjie.com:9092","kafka202.yinzhengjie.com:9092","kafka203.yinzhengjie.com:9092"] topic: "filebeat-ubuntu-syslog" partition.round_robin: reachable_only: true required_acks: 1 compression: gzip max_message_bytes: 1000000 processors: - add_host_metadata: ~ - add_cloud_metadata: ~ [root@es103.yinzhengjie.com ~]#
[root@es103.yinzhengjie.com ~]# systemctl restart filebeat
[root@es103.yinzhengjie.com ~]# tail -100f /var/log/filebeat/filebeat 2020-06-19T05:22:48.903Z INFO instance/beat.go:611 Home path: [/usr/share/filebeat] Config path: [/etc/filebeat] Data path: [/var/lib/filebeat] Logs path: [/var/log/filebeat] 2020-06-19T05:22:48.907Z INFO instance/beat.go:618 Beat UUID: 8f27e999-8ad7-4ee0-9b60-9d9331a7ba43 2020-06-19T05:22:48.907Z INFO [seccomp] seccomp/seccomp.go:116 Syscall filter successfully installed 2020-06-19T05:22:48.907Z INFO [beat] instance/beat.go:931 Beat info {"system_info": {"beat": {"path": {"config": "/etc/filebeat", "data": "/var/lib/filebeat", "home": "/ usr/share/filebeat", "logs": "/var/log/filebeat"}, "type": "filebeat", "uuid": "8f27e999-8ad7-4ee0-9b60-9d9331a7ba43"}}}2020-06-19T05:22:48.907Z INFO [beat] instance/beat.go:940 Build info {"system_info": {"build": {"commit": "75d698ad90ade7d497d9b3584654a3a2617e2f73", "libbeat": "6.8.9", "time": "2020-05-04T16:04:24.000Z", "version": "6.8.9"}}}2020-06-19T05:22:48.907Z INFO [beat] instance/beat.go:943 Go runtime info {"system_info": {"go": {"os":"linux","arch":"amd64","max_procs":2,"version":"go1.10.8"}}} 2020-06-19T05:22:48.908Z INFO [beat] instance/beat.go:947 Host info {"system_info": {"host": {"architecture":"x86_64","boot_time":"2020-06-19T04:51:34Z","containerized": false,"name":"es103.yinzhengjie.com","ip":["127.0.0.1/8","::1/128","172.200.5.103/21","fe80::20c:29ff:fe06:42e6/64"],"kernel_version":"4.15.0-106-generic","mac":["00:0c:29:06:42:e6"],"os":{"family":"debian","platform":"ubuntu","name":"Ubuntu","version":"18.04.3 LTS (Bionic Beaver)","major":18,"minor":4,"patch":3,"codename":"bionic"},"timezone":"UTC","timezone_offset_sec":0,"id":"8e0a6eabcd8c40f19da6b7f090c4cd2a"}}}2020-06-19T05:22:48.909Z INFO [beat] instance/beat.go:976 Process info {"system_info": {"process": {"capabilities": {"inheritable":null,"permitted":["chown","dac_override", "dac_read_search","fowner","fsetid","kill","setgid","setuid","setpcap","linux_immutable","net_bind_service","net_broadcast","net_admin","net_raw","ipc_lock","ipc_owner","sys_module","sys_rawio","sys_chroot","sys_ptrace","sys_pacct","sys_admin","sys_boot","sys_nice","sys_resource","sys_time","sys_tty_config","mknod","lease","audit_write","audit_control","setfcap","mac_override","mac_admin","syslog","wake_alarm","block_suspend","audit_read"],"effective":["chown","dac_override","dac_read_search","fowner","fsetid","kill","setgid","setuid","setpcap","linux_immutable","net_bind_service","net_broadcast","net_admin","net_raw","ipc_lock","ipc_owner","sys_module","sys_rawio","sys_chroot","sys_ptrace","sys_pacct","sys_admin","sys_boot","sys_nice","sys_resource","sys_time","sys_tty_config","mknod","lease","audit_write","audit_control","setfcap","mac_override","mac_admin","syslog","wake_alarm","block_suspend","audit_read"],"bounding":["chown","dac_override","dac_read_search","fowner","fsetid","kill","setgid","setuid","setpcap","linux_immutable","net_bind_service","net_broadcast","net_admin","net_raw","ipc_lock","ipc_owner","sys_module","sys_rawio","sys_chroot","sys_ptrace","sys_pacct","sys_admin","sys_boot","sys_nice","sys_resource","sys_time","sys_tty_config","mknod","lease","audit_write","audit_control","setfcap","mac_override","mac_admin","syslog","wake_alarm","block_suspend","audit_read"],"ambient":null}, "cwd": "/", "exe": "/usr/share/filebeat/bin/filebeat", "name": "filebeat", "pid": 2888, "ppid": 1, "seccomp": {"mode":"filter","no_new_privs":true}, "start_time": "2020-06-19T05:22:48.440Z"}}}2020-06-19T05:22:48.909Z INFO instance/beat.go:280 Setup Beat: filebeat; Version: 6.8.9 2020-06-19T05:22:48.911Z INFO [publisher] pipeline/module.go:110 Beat name: es103.yinzhengjie.com 2020-06-19T05:22:48.938Z INFO instance/beat.go:402 filebeat start running. 2020-06-19T05:22:48.938Z INFO registrar/registrar.go:134 Loading registrar data from /var/lib/filebeat/registry 2020-06-19T05:22:48.938Z INFO [monitoring] log/log.go:117 Starting metrics logging every 30s 2020-06-19T05:22:48.938Z INFO registrar/registrar.go:141 States Loaded from registrar: 2 2020-06-19T05:22:48.938Z WARN beater/filebeat.go:367 Filebeat is unable to load the Ingest Node pipelines for the configured modules because the Elasticsearch output is not confi gured/enabled. If you have already loaded the Ingest Node pipelines or are using Logstash pipelines, you can ignore this warning.2020-06-19T05:22:48.938Z INFO crawler/crawler.go:72 Loading Inputs: 2 2020-06-19T05:22:48.939Z INFO log/input.go:148 Configured paths: [/var/log/syslog] 2020-06-19T05:22:48.939Z INFO input/input.go:114 Starting input of type: log; ID: 5479846534217438201 2020-06-19T05:22:48.940Z INFO log/harvester.go:255 Harvester started for file: /var/log/syslog 2020-06-19T05:22:48.940Z INFO log/input.go:148 Configured paths: [/var/log/nginx/access.log] 2020-06-19T05:22:48.940Z INFO input/input.go:114 Starting input of type: log; ID: 1984074939396890659 2020-06-19T05:22:48.972Z INFO crawler/crawler.go:106 Loading and starting Inputs completed. Enabled inputs: 2 2020-06-19T05:22:48.972Z INFO cfgfile/reload.go:150 Config reloader started 2020-06-19T05:22:48.972Z INFO cfgfile/reload.go:205 Loading of config files completed. 2020-06-19T05:22:51.912Z INFO add_cloud_metadata/add_cloud_metadata.go:340 add_cloud_metadata: hosting provider type not detected. 2020-06-19T05:22:52.913Z INFO pipeline/output.go:95 Connecting to kafka(kafka201.yinzhengjie.com:9092,kafka202.yinzhengjie.com:9092,kafka203.yinzhengjie.com:9092) 2020-06-19T05:22:52.913Z INFO kafka/log.go:53 kafka message: Initializing new client 2020-06-19T05:22:52.913Z INFO kafka/log.go:53 client/metadata fetching metadata for all topics from broker kafka201.yinzhengjie.com:9092 2020-06-19T05:22:52.914Z INFO kafka/log.go:53 Connected to broker at kafka201.yinzhengjie.com:9092 (unregistered) 2020-06-19T05:22:52.917Z INFO kafka/log.go:53 client/brokers registered new broker #202 at kafka202.yinzhengjie.com:9092 2020-06-19T05:22:52.917Z INFO kafka/log.go:53 client/brokers registered new broker #201 at kafka201.yinzhengjie.com:9092 2020-06-19T05:22:52.917Z INFO kafka/log.go:53 client/brokers registered new broker #203 at kafka203.yinzhengjie.com:9092 2020-06-19T05:22:52.917Z INFO kafka/log.go:53 kafka message: Successfully initialized new client 2020-06-19T05:22:52.917Z INFO pipeline/output.go:105 Connection to kafka(kafka201.yinzhengjie.com:9092,kafka202.yinzhengjie.com:9092,kafka203.yinzhengjie.com:9092) established 2020-06-19T05:22:52.919Z INFO kafka/log.go:53 producer/broker/201 starting up 2020-06-19T05:22:52.919Z INFO kafka/log.go:53 producer/broker/201 state change to [open] on filebeat-ubuntu-syslog/0 2020-06-19T05:22:52.919Z INFO kafka/log.go:53 producer/broker/202 starting up 2020-06-19T05:22:52.919Z INFO kafka/log.go:53 producer/broker/202 state change to [open] on filebeat-ubuntu-syslog/2 2020-06-19T05:22:52.919Z INFO kafka/log.go:53 producer/broker/203 starting up 2020-06-19T05:22:52.919Z INFO kafka/log.go:53 producer/broker/203 state change to [open] on filebeat-ubuntu-syslog/1 2020-06-19T05:22:52.920Z INFO kafka/log.go:53 Connected to broker at kafka203.yinzhengjie.com:9092 (registered as #203) 2020-06-19T05:22:52.921Z INFO kafka/log.go:53 Connected to broker at kafka201.yinzhengjie.com:9092 (registered as #201) 2020-06-19T05:22:52.935Z INFO kafka/log.go:53 Connected to broker at kafka202.yinzhengjie.com:9092 (registered as #202) 2020-06-19T05:23:18.940Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":120,"time":{"ms" :126}},"total":{"ticks":130,"time":{"ms":144},"value":130},"user":{"ticks":10,"time":{"ms":18}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":11},"info":{"ephemeral_id":"b9005fb3-fe0b-4774-befd-cdbfa670605e","uptime":{"ms":30103}},"memstats":{"gc_next":7627184,"memory_alloc":4670944,"memory_total":8959296,"rss":27639808}},"filebeat":{"events":{"added":7,"done":7},"harvester":{"open_files":1,"running":1,"started":1}},"libbeat":{"config":{"module":{"running":0},"reloads":1},"output":{"events":{"acked":4,"batches":1,"total":4},"type":"kafka"},"outputs":{"kafka":{"bytes_read":4095,"bytes_write":2616}},"pipeline":{"clients":2,"events":{"active":0,"filtered":3,"published":4,"retry":4,"total":7},"queue":{"acked":4}}},"registrar":{"states":{"current":2,"update":7},"writes":{"success":4,"total":4}},"system":{"cpu":{"cores":2},"load":{"1":0.06,"15":0.28,"5":0.23,"norm":{"1":0.03,"15":0.14,"5":0.115}}}}}}2020-06-19T05:23:48.940Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":130,"time":{"ms" :8}},"total":{"ticks":140,"time":{"ms":9},"value":140},"user":{"ticks":10,"time":{"ms":1}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":11},"info":{"ephemeral_id":"b9005fb3-fe0b-4774-befd-cdbfa670605e","uptime":{"ms":60103}},"memstats":{"gc_next":7627184,"memory_alloc":4991736,"memory_total":9280088}},"filebeat":{"harvester":{"open_files":1,"running":1}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":2,"events":{"active":0}}},"registrar":{"states":{"current":2}},"system":{"load":{"1":0.04,"15":0.27,"5":0.21,"norm":{"1":0.02,"15":0.135,"5":0.105}}}}}}2020-06-19T05:24:18.948Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":140,"time":{"ms" :10}},"total":{"ticks":160,"time":{"ms":15},"value":160},"user":{"ticks":20,"time":{"ms":5}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":11},"info":{"ephemeral_id":"b9005fb3-fe0b-4774-befd-cdbfa670605e","uptime":{"ms":90110}},"memstats":{"gc_next":7627184,"memory_alloc":5454344,"memory_total":9742696,"rss":208896}},"filebeat":{"harvester":{"open_files":1,"running":1}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":2,"events":{"active":0}}},"registrar":{"states":{"current":2}},"system":{"load":{"1":0.02,"15":0.26,"5":0.18,"norm":{"1":0.01,"15":0.13,"5":0.09}}}}}}2020-06-19T05:24:28.952Z INFO log/harvester.go:255 Harvester started for file: /var/log/nginx/access.log 2020-06-19T05:24:48.940Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":150,"time":{"ms" :15}},"total":{"ticks":180,"time":{"ms":25},"value":180},"user":{"ticks":30,"time":{"ms":10}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":12},"info":{"ephemeral_id":"b9005fb3-fe0b-4774-befd-cdbfa670605e","uptime":{"ms":120104}},"memstats":{"gc_next":5734432,"memory_alloc":2900080,"memory_total":10936152,"rss":1650688}},"filebeat":{"events":{"added":2,"done":2},"harvester":{"open_files":2,"running":2,"started":1}},"libbeat":{"config":{"module":{"running":0}},"output":{"events":{"acked":1,"batches":1,"total":1}},"outputs":{"kafka":{"bytes_read":66,"bytes_write":821}},"pipeline":{"clients":2,"events":{"active":0,"filtered":1,"published":1,"total":2},"queue":{"acked":1}}},"registrar":{"states":{"current":2,"update":2},"writes":{"success":2,"total":2}},"system":{"load":{"1":0.01,"15":0.25,"5":0.17,"norm":{"1":0.005,"15":0.125,"5":0.085}}}}}}2020-06-19T05:25:18.940Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":160,"time":{"ms" :8}},"total":{"ticks":190,"time":{"ms":9},"value":190},"user":{"ticks":30,"time":{"ms":1}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":12},"info":{"ephemeral_id":"b9005fb3-fe0b-4774-befd-cdbfa670605e","uptime":{"ms":150103}},"memstats":{"gc_next":5734432,"memory_alloc":3290208,"memory_total":11326280}},"filebeat":{"harvester":{"open_files":2,"running":2}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":2,"events":{"active":0}}},"registrar":{"states":{"current":2}},"system":{"load":{"1":0.14,"15":0.26,"5":0.18,"norm":{"1":0.07,"15":0.13,"5":0.09}}}}}}2020-06-19T05:25:48.939Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":170,"time":{"ms" :6}},"total":{"ticks":200,"time":{"ms":7},"value":200},"user":{"ticks":30,"time":{"ms":1}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":12},"info":{"ephemeral_id":"b9005fb3-fe0b-4774-befd-cdbfa670605e","uptime":{"ms":180103}},"memstats":{"gc_next":5734432,"memory_alloc":3602912,"memory_total":11638984}},"filebeat":{"harvester":{"open_files":2,"running":2}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":2,"events":{"active":0}}},"registrar":{"states":{"current":2}},"system":{"load":{"1":0.09,"15":0.25,"5":0.16,"norm":{"1":0.045,"15":0.125,"5":0.08}}}}}}2020-06-19T05:26:18.940Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":170,"time":{"ms" :6}},"total":{"ticks":200,"time":{"ms":8},"value":200},"user":{"ticks":30,"time":{"ms":2}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":12},"info":{"ephemeral_id":"b9005fb3-fe0b-4774-befd-cdbfa670605e","uptime":{"ms":210103}},"memstats":{"gc_next":5734432,"memory_alloc":3823792,"memory_total":11859864}},"filebeat":{"harvester":{"open_files":2,"running":2}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":2,"events":{"active":0}}},"registrar":{"states":{"current":2}},"system":{"load":{"1":0.25,"15":0.26,"5":0.2,"norm":{"1":0.125,"15":0.13,"5":0.1}}}}}}2020-06-19T05:26:48.940Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":170},"total":{"t icks":210,"time":{"ms":10},"value":210},"user":{"ticks":40,"time":{"ms":10}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":12},"info":{"ephemeral_id":"b9005fb3-fe0b-4774-befd-cdbfa670605e","uptime":{"ms":240103}},"memstats":{"gc_next":4194304,"memory_alloc":1929568,"memory_total":12153976}},"filebeat":{"harvester":{"open_files":2,"running":2}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":2,"events":{"active":0}}},"registrar":{"states":{"current":2}},"system":{"load":{"1":0.22,"15":0.25,"5":0.19,"norm":{"1":0.11,"15":0.125,"5":0.095}}}}}}2020-06-19T05:27:18.939Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":180,"time":{"ms" :5}},"total":{"ticks":230,"time":{"ms":7},"value":230},"user":{"ticks":50,"time":{"ms":2}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":12},"info":{"ephemeral_id":"b9005fb3-fe0b-4774-befd-cdbfa670605e","uptime":{"ms":270103}},"memstats":{"gc_next":4194304,"memory_alloc":2320152,"memory_total":12544560}},"filebeat":{"harvester":{"open_files":2,"running":2}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":2,"events":{"active":0}}},"registrar":{"states":{"current":2}},"system":{"load":{"1":0.13,"15":0.24,"5":0.17,"norm":{"1":0.065,"15":0.12,"5":0.085}}}}}}2020-06-19T05:27:48.940Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":180,"time":{"ms" :6}},"total":{"ticks":230,"time":{"ms":7},"value":230},"user":{"ticks":50,"time":{"ms":1}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":12},"info":{"ephemeral_id":"b9005fb3-fe0b-4774-befd-cdbfa670605e","uptime":{"ms":300104}},"memstats":{"gc_next":4194304,"memory_alloc":2711320,"memory_total":12935728}},"filebeat":{"harvester":{"open_files":2,"running":2}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":2,"events":{"active":0}}},"registrar":{"states":{"current":2}},"system":{"load":{"1":0.08,"15":0.24,"5":0.15,"norm":{"1":0.04,"15":0.12,"5":0.075}}}}}}2020-06-19T05:27:56.944Z INFO log/harvester.go:280 File is inactive: /var/log/syslog. Closing because close_inactive of 5m0s reached. 2020-06-19T05:28:18.939Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":190,"time":{"ms" :8}},"total":{"ticks":240,"time":{"ms":11},"value":240},"user":{"ticks":50,"time":{"ms":3}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":11},"info":{"ephemeral_id":"b9005fb3-fe0b-4774-befd-cdbfa670605e","uptime":{"ms":330103}},"memstats":{"gc_next":4194304,"memory_alloc":2928392,"memory_total":13152800}},"filebeat":{"events":{"added":1,"done":1},"harvester":{"closed":1,"open_files":1,"running":1}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":2,"events":{"active":0,"filtered":1,"total":1}}},"registrar":{"states":{"current":2,"update":1},"writes":{"success":1,"total":1}},"system":{"load":{"1":0.05,"15":0.23,"5":0.14,"norm":{"1":0.025,"15":0.115,"5":0.07}}}}}}2020-06-19T05:28:48.939Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":200,"time":{"ms" :4}},"total":{"ticks":250,"time":{"ms":5},"value":250},"user":{"ticks":50,"time":{"ms":1}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":11},"info":{"ephemeral_id":"b9005fb3-fe0b-4774-befd-cdbfa670605e","uptime":{"ms":360103}},"memstats":{"gc_next":4194304,"memory_alloc":1981456,"memory_total":13462184}},"filebeat":{"harvester":{"open_files":1,"running":1}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":2,"events":{"active":0}}},"registrar":{"states":{"current":2}},"system":{"load":{"1":0.03,"15":0.22,"5":0.12,"norm":{"1":0.015,"15":0.11,"5":0.06}}}}}}
2>.配置logstash收集kafka数据到elasticsearch集群
[root@es103.yinzhengjie.com ~]# vim /etc/logstash/conf.d/kafka-to-elasticsearch.conf [root@es103.yinzhengjie.com ~]# [root@es103.yinzhengjie.com ~]# cat /etc/logstash/conf.d/kafka-to-elasticsearch.conf input { kafka { topics => ["filebeat-ubuntu-syslog"] bootstrap_servers => "kafka201.yinzhengjie.com:9092,kafka202.yinzhengjie.com:9092,kafka203.yinzhengjie.com:9092" codec => "json" group_id => "yinzhengjie-filebeat" decorate_events => true consumer_threads => 3 } } output{ if [fields][app] == "ubuntu-syslog" { elasticsearch { hosts => ["http://es101.yinzhengjie.com:9200","http://es102.yinzhengjie.com:9200"] index => "filebeat-syslog-172.200.5.103-%{+YYYY.MM.dd}" } } if [fields][app] == "nginx" { elasticsearch { hosts => ["http://es101.yinzhengjie.com:9200","http://es102.yinzhengjie.com:9200"] index => "filebeat-nginx-172.200.5.103-%{+YYYY.MM.dd}" } stdout { codec => "rubydebug" } } } [root@es103.yinzhengjie.com ~]#
[root@es103.yinzhengjie.com ~]# systemctl start logstash
[root@es103.yinzhengjie.com ~]# tail -100f /var/log/logstash/logstash-plain.log ssl.trustmanager.algorithm = PKIX ssl.truststore.location = null ssl.truststore.password = null ssl.truststore.type = JKS value.deserializer = class org.apache.kafka.common.serialization.StringDeserializer [2020-06-19T05:33:01,981][INFO ][org.apache.kafka.common.utils.AppInfoParser] Kafka version : 2.1.0 [2020-06-19T05:33:01,981][INFO ][org.apache.kafka.common.utils.AppInfoParser] Kafka commitId : eec43959745f444f [2020-06-19T05:33:01,991][INFO ][org.apache.kafka.clients.consumer.ConsumerConfig] ConsumerConfig values: auto.commit.interval.ms = 5000 auto.offset.reset = latest bootstrap.servers = [kafka201.yinzhengjie.com:9092, kafka202.yinzhengjie.com:9092, kafka203.yinzhengjie.com:9092] check.crcs = true client.dns.lookup = default client.id = logstash-2 connections.max.idle.ms = 540000 default.api.timeout.ms = 60000 enable.auto.commit = true exclude.internal.topics = true fetch.max.bytes = 52428800 fetch.max.wait.ms = 500 fetch.min.bytes = 1 group.id = yinzhengjie-filebeat heartbeat.interval.ms = 3000 interceptor.classes = [] internal.leave.group.on.close = true isolation.level = read_uncommitted key.deserializer = class org.apache.kafka.common.serialization.StringDeserializer max.partition.fetch.bytes = 1048576 max.poll.interval.ms = 300000 max.poll.records = 500 metadata.max.age.ms = 300000 metric.reporters = [] metrics.num.samples = 2 metrics.recording.level = INFO metrics.sample.window.ms = 30000 partition.assignment.strategy = [class org.apache.kafka.clients.consumer.RangeAssignor] receive.buffer.bytes = 65536 reconnect.backoff.max.ms = 1000 reconnect.backoff.ms = 50 request.timeout.ms = 30000 retry.backoff.ms = 100 sasl.client.callback.handler.class = null sasl.jaas.config = null sasl.kerberos.kinit.cmd = /usr/bin/kinit sasl.kerberos.min.time.before.relogin = 60000 sasl.kerberos.service.name = null sasl.kerberos.ticket.renew.jitter = 0.05 sasl.kerberos.ticket.renew.window.factor = 0.8 sasl.login.callback.handler.class = null sasl.login.class = null sasl.login.refresh.buffer.seconds = 300 sasl.login.refresh.min.period.seconds = 60 sasl.login.refresh.window.factor = 0.8 sasl.login.refresh.window.jitter = 0.05 sasl.mechanism = GSSAPI security.protocol = PLAINTEXT send.buffer.bytes = 131072 session.timeout.ms = 10000 ssl.cipher.suites = null ssl.enabled.protocols = [TLSv1.2, TLSv1.1, TLSv1] ssl.endpoint.identification.algorithm = https ssl.key.password = null ssl.keymanager.algorithm = SunX509 ssl.keystore.location = null ssl.keystore.password = null ssl.keystore.type = JKS ssl.protocol = TLS ssl.provider = null ssl.secure.random.implementation = null ssl.trustmanager.algorithm = PKIX ssl.truststore.location = null ssl.truststore.password = null ssl.truststore.type = JKS value.deserializer = class org.apache.kafka.common.serialization.StringDeserializer [2020-06-19T05:33:02,005][INFO ][org.apache.kafka.common.utils.AppInfoParser] Kafka version : 2.1.0 [2020-06-19T05:33:02,008][INFO ][org.apache.kafka.common.utils.AppInfoParser] Kafka commitId : eec43959745f444f [2020-06-19T05:33:02,233][INFO ][org.apache.kafka.clients.Metadata] Cluster ID: aDBW0CNdQeC8lxYFd77sxg [2020-06-19T05:33:02,235][INFO ][org.apache.kafka.clients.Metadata] Cluster ID: aDBW0CNdQeC8lxYFd77sxg [2020-06-19T05:33:02,234][INFO ][org.apache.kafka.clients.Metadata] Cluster ID: aDBW0CNdQeC8lxYFd77sxg [2020-06-19T05:33:02,241][INFO ][org.apache.kafka.clients.consumer.internals.AbstractCoordinator] [Consumer clientId=logstash-2, groupId=yinzhengjie-filebeat] Discovered group coordinator k afka203.yinzhengjie.com:9092 (id: 2147483444 rack: null)[2020-06-19T05:33:02,242][INFO ][org.apache.kafka.clients.consumer.internals.AbstractCoordinator] [Consumer clientId=logstash-0, groupId=yinzhengjie-filebeat] Discovered group coordinator k afka203.yinzhengjie.com:9092 (id: 2147483444 rack: null)[2020-06-19T05:33:02,242][INFO ][org.apache.kafka.clients.consumer.internals.AbstractCoordinator] [Consumer clientId=logstash-1, groupId=yinzhengjie-filebeat] Discovered group coordinator k afka203.yinzhengjie.com:9092 (id: 2147483444 rack: null)[2020-06-19T05:33:02,250][INFO ][org.apache.kafka.clients.consumer.internals.ConsumerCoordinator] [Consumer clientId=logstash-2, groupId=yinzhengjie-filebeat] Revoking previously assigned p artitions [][2020-06-19T05:33:02,251][INFO ][org.apache.kafka.clients.consumer.internals.AbstractCoordinator] [Consumer clientId=logstash-2, groupId=yinzhengjie-filebeat] (Re-)joining group [2020-06-19T05:33:02,252][INFO ][org.apache.kafka.clients.consumer.internals.ConsumerCoordinator] [Consumer clientId=logstash-0, groupId=yinzhengjie-filebeat] Revoking previously assigned p artitions [][2020-06-19T05:33:02,252][INFO ][org.apache.kafka.clients.consumer.internals.AbstractCoordinator] [Consumer clientId=logstash-0, groupId=yinzhengjie-filebeat] (Re-)joining group [2020-06-19T05:33:02,254][INFO ][org.apache.kafka.clients.consumer.internals.ConsumerCoordinator] [Consumer clientId=logstash-1, groupId=yinzhengjie-filebeat] Revoking previously assigned p artitions [][2020-06-19T05:33:02,254][INFO ][org.apache.kafka.clients.consumer.internals.AbstractCoordinator] [Consumer clientId=logstash-1, groupId=yinzhengjie-filebeat] (Re-)joining group [2020-06-19T05:33:02,373][INFO ][logstash.agent ] Successfully started Logstash API endpoint {:port=>9600} [2020-06-19T05:33:08,291][INFO ][org.apache.kafka.clients.consumer.internals.AbstractCoordinator] [Consumer clientId=logstash-1, groupId=yinzhengjie-filebeat] Successfully joined group with generation 1[2020-06-19T05:33:08,293][INFO ][org.apache.kafka.clients.consumer.internals.AbstractCoordinator] [Consumer clientId=logstash-2, groupId=yinzhengjie-filebeat] Successfully joined group with generation 1[2020-06-19T05:33:08,293][INFO ][org.apache.kafka.clients.consumer.internals.AbstractCoordinator] [Consumer clientId=logstash-0, groupId=yinzhengjie-filebeat] Successfully joined group with generation 1[2020-06-19T05:33:08,297][INFO ][org.apache.kafka.clients.consumer.internals.ConsumerCoordinator] [Consumer clientId=logstash-2, groupId=yinzhengjie-filebeat] Setting newly assigned partiti ons [filebeat-ubuntu-syslog-2][2020-06-19T05:33:08,297][INFO ][org.apache.kafka.clients.consumer.internals.ConsumerCoordinator] [Consumer clientId=logstash-1, groupId=yinzhengjie-filebeat] Setting newly assigned partiti ons [filebeat-ubuntu-syslog-1][2020-06-19T05:33:08,299][INFO ][org.apache.kafka.clients.consumer.internals.ConsumerCoordinator] [Consumer clientId=logstash-0, groupId=yinzhengjie-filebeat] Setting newly assigned partiti ons [filebeat-ubuntu-syslog-0][2020-06-19T05:33:08,333][INFO ][org.apache.kafka.clients.consumer.internals.Fetcher] [Consumer clientId=logstash-2, groupId=yinzhengjie-filebeat] Resetting offset for partition filebeat-ub untu-syslog-2 to offset 185632.[2020-06-19T05:33:08,335][INFO ][org.apache.kafka.clients.consumer.internals.Fetcher] [Consumer clientId=logstash-0, groupId=yinzhengjie-filebeat] Resetting offset for partition filebeat-ub untu-syslog-0 to offset 185634.[2020-06-19T05:33:08,334][INFO ][org.apache.kafka.clients.consumer.internals.Fetcher] [Consumer clientId=logstash-1, groupId=yinzhengjie-filebeat] Resetting offset for partition filebeat-ub untu-syslog-1 to offset 185632.
3>.浏览器访问nginx并观察kibana的变化
五.filebeat结合redis收集不同主机的多类型日志
1>.角色分配说明
es101.yinzhengjie.com:
Elasticsearch，kibana角色
es102.yinzhengjie.com:
Elasticsearch角色
es103.yinzhengjie.com:
Elasticsearch，Nginx，Filebeat，Logstash角色
redis104.yinzhengjie.com:
Redis角色
logstash105.yinzhengjie.com:
Logstash角色
2>.配置filebeat收集日志到Redis
[root@es103.yinzhengjie.com ~]# vim /etc/filebeat/filebeat.yml [root@es103.yinzhengjie.com ~]# [root@es103.yinzhengjie.com ~]# egrep -v "^*#|^$" /etc/filebeat/filebeat.yml filebeat.inputs: - type: log enabled: true paths: - /var/log/syslog fields: host: "172.200.5.103" type: "filebeat-syslog-172-200-5-103" app: "ubuntu-syslog" - type: log enable: true paths: - /var/log/nginx/access.log fields: host: "172.200.5.103" type: "filebeat-nginx-accesslog-172-200-5-103" app: "nginx" filebeat.config.modules: path: ${path.config}/modules.d/*.yml reload.enabled: false setup.template.enabled: false setup.template.settings: index.number_of_shards: 3 setup.kibana: output.redis: hosts: ["redis104.yinzhengjie.com"] port: 6379 password: "yinzhengjie" key: "filebeat-log-172-200-5-103" db: 0 timeout: 10 processors: - add_host_metadata: ~ - add_cloud_metadata: ~ [root@es103.yinzhengjie.com ~]#
[root@es103.yinzhengjie.com ~]# systemctl restart filebeat
[root@es103.yinzhengjie.com ~]# tail -100f /var/log/filebeat/filebeat 2020-06-22T04:41:55.115Z INFO instance/beat.go:611 Home path: [/usr/share/filebeat] Config path: [/etc/filebeat] Data path: [/var/lib/filebeat] Logs path: [/var/log/filebeat] 2020-06-22T04:41:55.116Z INFO instance/beat.go:618 Beat UUID: 8f27e999-8ad7-4ee0-9b60-9d9331a7ba43 2020-06-22T04:41:55.117Z INFO [seccomp] seccomp/seccomp.go:116 Syscall filter successfully installed 2020-06-22T04:41:55.117Z INFO [beat] instance/beat.go:931 Beat info {"system_info": {"beat": {"path": {"config": "/etc/filebeat", "data": "/var/lib/filebeat", "home": "/ usr/share/filebeat", "logs": "/var/log/filebeat"}, "type": "filebeat", "uuid": "8f27e999-8ad7-4ee0-9b60-9d9331a7ba43"}}}2020-06-22T04:41:55.117Z INFO [beat] instance/beat.go:940 Build info {"system_info": {"build": {"commit": "75d698ad90ade7d497d9b3584654a3a2617e2f73", "libbeat": "6.8.9", "time": "2020-05-04T16:04:24.000Z", "version": "6.8.9"}}}2020-06-22T04:41:55.117Z INFO [beat] instance/beat.go:943 Go runtime info {"system_info": {"go": {"os":"linux","arch":"amd64","max_procs":2,"version":"go1.10.8"}}} 2020-06-22T04:41:55.118Z INFO [beat] instance/beat.go:947 Host info {"system_info": {"host": {"architecture":"x86_64","boot_time":"2020-06-22T03:29:20Z","containerized": false,"name":"es103.yinzhengjie.com","ip":["127.0.0.1/8","::1/128","172.200.5.103/21","fe80::20c:29ff:fe06:42e6/64"],"kernel_version":"4.15.0-106-generic","mac":["00:0c:29:06:42:e6"],"os":{"family":"debian","platform":"ubuntu","name":"Ubuntu","version":"18.04.3 LTS (Bionic Beaver)","major":18,"minor":4,"patch":3,"codename":"bionic"},"timezone":"UTC","timezone_offset_sec":0,"id":"8e0a6eabcd8c40f19da6b7f090c4cd2a"}}}2020-06-22T04:41:55.119Z INFO [beat] instance/beat.go:976 Process info {"system_info": {"process": {"capabilities": {"inheritable":null,"permitted":["chown","dac_override", "dac_read_search","fowner","fsetid","kill","setgid","setuid","setpcap","linux_immutable","net_bind_service","net_broadcast","net_admin","net_raw","ipc_lock","ipc_owner","sys_module","sys_rawio","sys_chroot","sys_ptrace","sys_pacct","sys_admin","sys_boot","sys_nice","sys_resource","sys_time","sys_tty_config","mknod","lease","audit_write","audit_control","setfcap","mac_override","mac_admin","syslog","wake_alarm","block_suspend","audit_read"],"effective":["chown","dac_override","dac_read_search","fowner","fsetid","kill","setgid","setuid","setpcap","linux_immutable","net_bind_service","net_broadcast","net_admin","net_raw","ipc_lock","ipc_owner","sys_module","sys_rawio","sys_chroot","sys_ptrace","sys_pacct","sys_admin","sys_boot","sys_nice","sys_resource","sys_time","sys_tty_config","mknod","lease","audit_write","audit_control","setfcap","mac_override","mac_admin","syslog","wake_alarm","block_suspend","audit_read"],"bounding":["chown","dac_override","dac_read_search","fowner","fsetid","kill","setgid","setuid","setpcap","linux_immutable","net_bind_service","net_broadcast","net_admin","net_raw","ipc_lock","ipc_owner","sys_module","sys_rawio","sys_chroot","sys_ptrace","sys_pacct","sys_admin","sys_boot","sys_nice","sys_resource","sys_time","sys_tty_config","mknod","lease","audit_write","audit_control","setfcap","mac_override","mac_admin","syslog","wake_alarm","block_suspend","audit_read"],"ambient":null}, "cwd": "/", "exe": "/usr/share/filebeat/bin/filebeat", "name": "filebeat", "pid": 5575, "ppid": 1, "seccomp": {"mode":"filter","no_new_privs":true}, "start_time": "2020-06-22T04:41:54.979Z"}}}2020-06-22T04:41:55.119Z INFO instance/beat.go:280 Setup Beat: filebeat; Version: 6.8.9 2020-06-22T04:41:55.119Z WARN [cfgwarn] redis/redis.go:61 DEPRECATED: The Redis outputs port setting Will be removed in version: 7.0.0 2020-06-22T04:41:55.121Z INFO [publisher] pipeline/module.go:110 Beat name: es103.yinzhengjie.com 2020-06-22T04:41:55.122Z INFO instance/beat.go:402 filebeat start running. 2020-06-22T04:41:55.122Z INFO registrar/registrar.go:134 Loading registrar data from /var/lib/filebeat/registry 2020-06-22T04:41:55.123Z INFO registrar/registrar.go:141 States Loaded from registrar: 2 2020-06-22T04:41:55.123Z WARN beater/filebeat.go:367 Filebeat is unable to load the Ingest Node pipelines for the configured modules because the Elasticsearch output is not confi gured/enabled. If you have already loaded the Ingest Node pipelines or are using Logstash pipelines, you can ignore this warning.2020-06-22T04:41:55.123Z INFO crawler/crawler.go:72 Loading Inputs: 2 2020-06-22T04:41:55.124Z INFO [monitoring] log/log.go:117 Starting metrics logging every 30s 2020-06-22T04:41:55.130Z INFO log/input.go:148 Configured paths: [/var/log/syslog] 2020-06-22T04:41:55.130Z INFO input/input.go:114 Starting input of type: log; ID: 5479846534217438201 2020-06-22T04:41:55.131Z INFO log/harvester.go:255 Harvester started for file: /var/log/syslog 2020-06-22T04:41:55.133Z INFO log/input.go:148 Configured paths: [/var/log/nginx/access.log] 2020-06-22T04:41:55.133Z INFO input/input.go:114 Starting input of type: log; ID: 1984074939396890659 2020-06-22T04:41:55.135Z INFO crawler/crawler.go:106 Loading and starting Inputs completed. Enabled inputs: 2 2020-06-22T04:41:55.135Z INFO cfgfile/reload.go:150 Config reloader started 2020-06-22T04:41:55.135Z INFO cfgfile/reload.go:205 Loading of config files completed. 2020-06-22T04:41:55.135Z INFO log/harvester.go:255 Harvester started for file: /var/log/nginx/access.log 2020-06-22T04:41:58.120Z INFO add_cloud_metadata/add_cloud_metadata.go:340 add_cloud_metadata: hosting provider type not detected. 2020-06-22T04:41:58.240Z INFO pipeline/output.go:95 Connecting to redis(tcp://redis104.yinzhengjie.com:6379) 2020-06-22T04:41:58.245Z INFO pipeline/output.go:105 Connection to redis(tcp://redis104.yinzhengjie.com:6379) established 2020-06-22T04:42:25.127Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":130,"time":{"ms" :137}},"total":{"ticks":240,"time":{"ms":249},"value":240},"user":{"ticks":110,"time":{"ms":112}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":9},"info":{"ephemeral_id":"ebc41be6-c312-46bb-969f-16210bb164f0","uptime":{"ms":30034}},"memstats":{"gc_next":4825600,"memory_alloc":2419688,"memory_total":28174640,"rss":38600704}},"filebeat":{"events":{"added":2357,"done":2357},"harvester":{"open_files":2,"running":2,"started":2}},"libbeat":{"config":{"module":{"running":0},"reloads":1},"output":{"events":{"acked":2353,"batches":9,"total":2353},"read":{"bytes":2811},"type":"redis","write":{"bytes":1894223}},"pipeline":{"clients":2,"events":{"active":0,"filtered":4,"published":2353,"retry":2048,"total":2357},"queue":{"acked":2353}}},"registrar":{"states":{"current":2,"update":2357},"writes":{"success":13,"total":13}},"system":{"cpu":{"cores":2},"load":{"1":0.35,"15":0.12,"5":0.3,"norm":{"1":0.175,"15":0.06,"5":0.15}}}}}}2020-06-22T04:42:55.126Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":210,"time":{"ms" :78}},"total":{"ticks":370,"time":{"ms":134},"value":370},"user":{"ticks":160,"time":{"ms":56}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":9},"info":{"ephemeral_id":"ebc41be6-c312-46bb-969f-16210bb164f0","uptime":{"ms":60034}},"memstats":{"gc_next":4856464,"memory_alloc":2478224,"memory_total":30272256}},"filebeat":{"events":{"active":3,"added":30,"done":27},"harvester":{"open_files":2,"running":2}},"libbeat":{"config":{"module":{"running":0}},"output":{"events":{"acked":27,"batches":8,"total":27},"read":{"bytes":56},"write":{"bytes":27195}},"pipeline":{"clients":2,"events":{"active":3,"published":30,"total":30},"queue":{"acked":27}}},"registrar":{"states":{"current":2,"update":27},"writes":{"success":8,"total":8}},"system":{"load":{"1":0.21,"15":0.12,"5":0.27,"norm":{"1":0.105,"15":0.06,"5":0.135}}}}}}2020-06-22T04:43:25.127Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":240,"time":{"ms" :27}},"total":{"ticks":500,"time":{"ms":127},"value":500},"user":{"ticks":260,"time":{"ms":100}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":9},"info":{"ephemeral_id":"ebc41be6-c312-46bb-969f-16210bb164f0","uptime":{"ms":90034}},"memstats":{"gc_next":4852608,"memory_alloc":2904816,"memory_total":32731800,"rss":-40960}},"filebeat":{"events":{"added":32,"done":32},"harvester":{"open_files":2,"running":2}},"libbeat":{"config":{"module":{"running":0}},"output":{"events":{"acked":32,"batches":10,"total":32},"read":{"bytes":70},"write":{"bytes":32256}},"pipeline":{"clients":2,"events":{"active":3,"published":32,"total":32},"queue":{"acked":32}}},"registrar":{"states":{"current":2,"update":32},"writes":{"success":10,"total":10}},"system":{"load":{"1":0.13,"15":0.11,"5":0.24,"norm":{"1":0.065,"15":0.055,"5":0.12}}}}}}2020-06-22T04:43:55.130Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":270,"time":{"ms" :30}},"total":{"ticks":560,"time":{"ms":54},"value":560},"user":{"ticks":290,"time":{"ms":24}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":9},"info":{"ephemeral_id":"ebc41be6-c312-46bb-969f-16210bb164f0","uptime":{"ms":120038}},"memstats":{"gc_next":4933952,"memory_alloc":3110584,"memory_total":34960008}},"filebeat":{"events":{"active":-3,"added":29,"done":32},"harvester":{"open_files":2,"running":2}},"libbeat":{"config":{"module":{"running":0}},"output":{"events":{"acked":32,"batches":9,"total":32},"read":{"bytes":63},"write":{"bytes":32209}},"pipeline":{"clients":2,"events":{"active":0,"published":29,"total":29},"queue":{"acked":32}}},"registrar":{"states":{"current":2,"update":32},"writes":{"success":9,"total":9}},"system":{"load":{"1":0.08,"15":0.1,"5":0.22,"norm":{"1":0.04,"15":0.05,"5":0.11}}}}}}2020-06-22T04:44:25.132Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":330,"time":{"ms" :62}},"total":{"ticks":620,"time":{"ms":65},"value":620},"user":{"ticks":290,"time":{"ms":3}}},"handles":{"limit":{"hard":4096,"soft":1024},"open":9},"info":{"ephemeral_id":"ebc41be6-c312-46bb-969f-16210bb164f0","uptime":{"ms":150040}},"memstats":{"gc_next":4879968,"memory_alloc":3123384,"memory_total":37225192}},"filebeat":{"events":{"added":30,"done":30},"harvester":{"open_files":2,"running":2}},"libbeat":{"config":{"module":{"running":0}},"output":{"events":{"acked":30,"batches":9,"total":30},"read":{"bytes":63},"write":{"bytes":30222}},"pipeline":{"clients":2,"events":{"active":0,"published":30,"total":30},"queue":{"acked":30}}},"registrar":{"states":{"current":2,"update":30},"writes":{"success":9,"total":9}},"system":{"load":{"1":0.04,"15":0.1,"5":0.2,"norm":{"1":0.02,"15":0.05,"5":0.1}}}}}}
3>.在redis上验证数据是否被成功写入
[root@redis104.yinzhengjie.com ~]# redis-cli -h 172.200.5.104 172.200.5.104:6379> AUTH yinzhengjie OK 172.200.5.104:6379> 172.200.5.104:6379> KEYS * 1) "filebeat-log-172-200-5-103" 172.200.5.104:6379> 172.200.5.104:6379> LPOP filebeat-log-172-200-5-103 "{"@timestamp":"2020-06-22T04:41:55.135Z","@metadata":{"beat":"filebeat","type":"doc","version":"6.8.9"},"offset":5306,"log":{"file":{"path":"/var/log/nginx/acces s.log"}},"fields":{"type":"filebeat-nginx-accesslog-172-200-5-103","app":"nginx","host":"172.200.5.103"},"message":"{\"@timestamp\":\"2020-06-22T04:40:28+00:00\",\"host\":\"172.200.5.103\",\"clientip\":\"172.200.5.103\",\"size\":612,\"responsetime\":0.000,\"upstreamtime\":\"-\",\"upstreamhost\":\"-\",\"http_host\":\"es103.yinzhengjie.com\",\"uri\":\"/index.nginx-debian.html\",\"domain\":\"es103.yinzhengjie.com\",\"xff\":\"-\",\"referer\":\"-\",\"tcp_xff\":\"\",\"http_user_agent\":\"curl/7.58.0\",\"status\":\"200\"}","source":"/var/log/nginx/access.log","beat":{"name":"es103.yinzhengjie.com","hostname":"es103.yinzhengjie.com","version":"6.8.9"},"host":{"name":"es103.yinzhengjie.com","architecture":"x86_64","os":{"platform":"ubuntu","version":"18.04.3 LTS (Bionic Beaver)","family":"debian","name":"Ubuntu","codename":"bionic"},"id":"8e0a6eabcd8c40f19da6b7f090c4cd2a","containerized":false},"prospector":{"type":"log"},"input":{"type":"log"}}"172.200.5.104:6379>
4>.在logstash节点收集数据到elasticsearch集群
[root@logstash105.yinzhengjie.com ~]# vim /etc/logstash/conf.d/redis-to-elasticsearch.conf [root@logstash105.yinzhengjie.com ~]# [root@logstash105.yinzhengjie.com ~]# cat /etc/logstash/conf.d/redis-to-elasticsearch.conf input { redis { host => "redis104.yinzhengjie.com" port => "6379" password => "yinzhengjie" db => "0" key => "filebeat-log-172-200-5-103" data_type => "list" codec => "json" } } output { if [fields][app] == "ubuntu-syslog" { elasticsearch { hosts => ["http://es101.yinzhengjie.com:9200","http://es102.yinzhengjie.com:9200"] index => "logstash-syslog-172.200.5.103-%{+YYYY.MM.dd}" } } if [fields][app] == "nginx" { elasticsearch { hosts => ["http://es101.yinzhengjie.com:9200","http://es102.yinzhengjie.com:9200"] index => "logstash-nginx-172.200.5.103-%{+YYYY.MM.dd}" } } } [root@logstash105.yinzhengjie.com ~]#
[root@logstash105.yinzhengjie.com ~]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/redis-to-elasticsearch.conf -t WARNING: Could not find logstash.yml which is typically located in $LS_HOME/config or /etc/logstash. You can specify the path using --path.settings. Continuing using the defaults Could not find log4j2 configuration at path /usr/share/logstash/config/log4j2.properties. Using default config which logs errors to the console [INFO ] 2020-06-22 05:02:18.597 [main] writabledirectory - Creating directory {:setting=>"path.queue", :path=>"/usr/share/logstash/data/queue"} [INFO ] 2020-06-22 05:02:18.614 [main] writabledirectory - Creating directory {:setting=>"path.dead_letter_queue", :path=>"/usr/share/logstash/data/dead_letter_queue"} [WARN ] 2020-06-22 05:02:18.927 [LogStash::Runner] multilocal - Ignoring the 'pipelines.yml' file because modules or command line options are specified Configuration OK [INFO ] 2020-06-22 05:02:24.270 [LogStash::Runner] runner - Using config.test_and_exit mode. Config Validation Result: OK. Exiting Logstash [root@logstash105.yinzhengjie.com ~]#
[root@logstash105.yinzhengjie.com ~]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/redis-to-elasticsearch.conf WARNING: Could not find logstash.yml which is typically located in $LS_HOME/config or /etc/logstash. You can specify the path using --path.settings. Continuing using the defaults Could not find log4j2 configuration at path /usr/share/logstash/config/log4j2.properties. Using default config which logs errors to the console [WARN ] 2020-06-22 05:03:20.132 [LogStash::Runner] multilocal - Ignoring the 'pipelines.yml' file because modules or command line options are specified [INFO ] 2020-06-22 05:03:20.145 [LogStash::Runner] runner - Starting Logstash {"logstash.version"=>"6.8.9"} [INFO ] 2020-06-22 05:03:20.167 [LogStash::Runner] agent - No persistent UUID file found. Generating new UUID {:uuid=>"dbc3f0d0-86ac-4ad4-bc57-7c834521ba6b", :path=>"/usr/share/logstash/dat a/uuid"}[INFO ] 2020-06-22 05:03:25.217 [Converge PipelineAction::Create<main>] pipeline - Starting pipeline {:pipeline_id=>"main", "pipeline.workers"=>2, "pipeline.batch.size"=>125, "pipeline.batc h.delay"=>50}[INFO ] 2020-06-22 05:03:25.797 [[main]-pipeline-manager] elasticsearch - Elasticsearch pool URLs updated {:changes=>{:removed=>[], :added=>[http://es101.yinzhengjie.com:9200/, http://es102 .yinzhengjie.com:9200/]}}[WARN ] 2020-06-22 05:03:25.960 [[main]-pipeline-manager] elasticsearch - Restored connection to ES instance {:url=>"http://es101.yinzhengjie.com:9200/"} [INFO ] 2020-06-22 05:03:26.122 [[main]-pipeline-manager] elasticsearch - ES Output version determined {:es_version=>6} [WARN ] 2020-06-22 05:03:26.124 [[main]-pipeline-manager] elasticsearch - Detected a 6.x and above cluster: the `type` event field won't be used to determine the document _type {:es_version =>6}[WARN ] 2020-06-22 05:03:26.134 [[main]-pipeline-manager] elasticsearch - Restored connection to ES instance {:url=>"http://es102.yinzhengjie.com:9200/"} [INFO ] 2020-06-22 05:03:26.177 [[main]-pipeline-manager] elasticsearch - New Elasticsearch output {:class=>"LogStash::Outputs::ElasticSearch", :hosts=>["http://es101.yinzhengjie.com:9200", "http://es102.yinzhengjie.com:9200"]}[INFO ] 2020-06-22 05:03:26.211 [[main]-pipeline-manager] elasticsearch - Elasticsearch pool URLs updated {:changes=>{:removed=>[], :added=>[http://es101.yinzhengjie.com:9200/, http://es102 .yinzhengjie.com:9200/]}}[INFO ] 2020-06-22 05:03:26.217 [Ruby-0-Thread-5: :1] elasticsearch - Using default mapping template [WARN ] 2020-06-22 05:03:26.236 [[main]-pipeline-manager] elasticsearch - Restored connection to ES instance {:url=>"http://es101.yinzhengjie.com:9200/"} [INFO ] 2020-06-22 05:03:26.245 [[main]-pipeline-manager] elasticsearch - ES Output version determined {:es_version=>6} [WARN ] 2020-06-22 05:03:26.246 [[main]-pipeline-manager] elasticsearch - Detected a 6.x and above cluster: the `type` event field won't be used to determine the document _type {:es_version =>6}[WARN ] 2020-06-22 05:03:26.251 [[main]-pipeline-manager] elasticsearch - Restored connection to ES instance {:url=>"http://es102.yinzhengjie.com:9200/"} [INFO ] 2020-06-22 05:03:26.262 [Ruby-0-Thread-5: :1] elasticsearch - Attempting to install template {:manage_template=>{"template"=>"logstash-*", "version"=>60001, "settings"=>{"index.refr esh_interval"=>"5s"}, "mappings"=>{"_default_"=>{"dynamic_templates"=>[{"message_field"=>{"path_match"=>"message", "match_mapping_type"=>"string", "mapping"=>{"type"=>"text", "norms"=>false}}}, {"string_fields"=>{"match"=>"*", "match_mapping_type"=>"string", "mapping"=>{"type"=>"text", "norms"=>false, "fields"=>{"keyword"=>{"type"=>"keyword", "ignore_above"=>256}}}}}], "properties"=>{"@timestamp"=>{"type"=>"date"}, "@version"=>{"type"=>"keyword"}, "geoip"=>{"dynamic"=>true, "properties"=>{"ip"=>{"type"=>"ip"}, "location"=>{"type"=>"geo_point"}, "latitude"=>{"type"=>"half_float"}, "longitude"=>{"type"=>"half_float"}}}}}}}}[INFO ] 2020-06-22 05:03:26.284 [[main]-pipeline-manager] elasticsearch - New Elasticsearch output {:class=>"LogStash::Outputs::ElasticSearch", :hosts=>["http://es101.yinzhengjie.com:9200", "http://es102.yinzhengjie.com:9200"]}[INFO ] 2020-06-22 05:03:26.286 [Ruby-0-Thread-7: :1] elasticsearch - Using default mapping template [INFO ] 2020-06-22 05:03:26.290 [Ruby-0-Thread-7: :1] elasticsearch - Attempting to install template {:manage_template=>{"template"=>"logstash-*", "version"=>60001, "settings"=>{"index.refr esh_interval"=>"5s"}, "mappings"=>{"_default_"=>{"dynamic_templates"=>[{"message_field"=>{"path_match"=>"message", "match_mapping_type"=>"string", "mapping"=>{"type"=>"text", "norms"=>false}}}, {"string_fields"=>{"match"=>"*", "match_mapping_type"=>"string", "mapping"=>{"type"=>"text", "norms"=>false, "fields"=>{"keyword"=>{"type"=>"keyword", "ignore_above"=>256}}}}}], "properties"=>{"@timestamp"=>{"type"=>"date"}, "@version"=>{"type"=>"keyword"}, "geoip"=>{"dynamic"=>true, "properties"=>{"ip"=>{"type"=>"ip"}, "location"=>{"type"=>"geo_point"}, "latitude"=>{"type"=>"half_float"}, "longitude"=>{"type"=>"half_float"}}}}}}}}[INFO ] 2020-06-22 05:03:26.366 [[main]-pipeline-manager] redis - Registering Redis {:identity=>"redis://<password>@redis104.yinzhengjie.com:6379/0 list:filebeat-log-172-200-5-103"} [INFO ] 2020-06-22 05:03:26.377 [Converge PipelineAction::Create<main>] pipeline - Pipeline started successfully {:pipeline_id=>"main", :thread=>"#<Thread:0x5c1f2609 run>"} [INFO ] 2020-06-22 05:03:26.463 [Ruby-0-Thread-1: /usr/share/logstash/lib/bootstrap/environment.rb:6] agent - Pipelines running {:count=>1, :running_pipelines=>[:main], :non_running_pipelin es=>[]}[INFO ] 2020-06-22 05:03:26.955 [Api Webserver] agent - Successfully started Logstash API endpoint {:port=>9600}
[root@logstash105.yinzhengjie.com ~]# systemctl start logstash #以守护进程方式启动logstash，注意观察"/var/log/syslog"日志信息
5>.在kibana配置索引并查看数据
六.filebeat结合logstash收集不同主机的多类型日志
1>.角色分配说明
es101.yinzhengjie.com:
Elasticsearch，kibana角色
es102.yinzhengjie.com:
Elasticsearch角色
es103.yinzhengjie.com:
Elasticsearch，Nginx，Filebeat，Logstash角色
redis104.yinzhengjie.com:
Redis角色
logstash105.yinzhengjie.com:
Logstash角色
2>.filebeat将数据写入到logstash
[root@es103.yinzhengjie.com ~]# vim /etc/filebeat/filebeat.yml [root@es103.yinzhengjie.com ~]# [root@es103.yinzhengjie.com ~]# egrep -v "^*#|^$" /etc/filebeat/filebeat.yml filebeat.inputs: - type: log enabled: true paths: - /var/log/syslog fields: host: "172.200.5.103" type: "filebeat-syslog-172-200-5-103" app: "ubuntu-syslog" - type: log enable: true paths: - /var/log/nginx/access.log fields: host: "172.200.5.103" type: "filebeat-nginx-accesslog-172-200-5-103" app: "nginx" filebeat.config.modules: path: ${path.config}/modules.d/*.yml reload.enabled: false setup.template.enabled: false setup.template.settings: index.number_of_shards: 3 setup.kibana: output.logstash: hosts: ["logstash105.yinzhengjie.com:8888"] processors: - add_host_metadata: ~ - add_cloud_metadata: ~ [root@es103.yinzhengjie.com ~]#
[root@es103.yinzhengjie.com ~]# systemctl restart filebeat.service
3>.logstash将数据写入到redis
[root@logstash105.yinzhengjie.com ~]# vim /etc/logstash/conf.d/beats-to-redis.conf [root@logstash105.yinzhengjie.com ~]# [root@logstash105.yinzhengjie.com ~]# cat /etc/logstash/conf.d/beats-to-redis.conf input { beats { host => "logstash105.yinzhengjie.com" port => 8888 codec => "json" } } output { if [fields][app] == "ubuntu-syslog" { redis { host => "redis104.yinzhengjie.com" port => "6379" password => "yinzhengjie" db => "0" key => "filebeat-syslog-172-200-5-103" data_type => "list" codec => "json" } } if [fields][app] == "nginx" { redis { host => "redis104.yinzhengjie.com" port => "6379" password => "yinzhengjie" db => "0" key => "filebeat-nginx-172-200-5-103" data_type => "list" codec => "json" } } } [root@logstash105.yinzhengjie.com ~]#
[root@logstash105.yinzhengjie.com ~]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/beats-to-redis.conf -t WARNING: Could not find logstash.yml which is typically located in $LS_HOME/config or /etc/logstash. You can specify the path using --path.settings. Continuing using the defaults Could not find log4j2 configuration at path /usr/share/logstash/config/log4j2.properties. Using default config which logs errors to the console [WARN ] 2020-06-22 05:46:04.321 [LogStash::Runner] multilocal - Ignoring the 'pipelines.yml' file because modules or command line options are specified Configuration OK [INFO ] 2020-06-22 05:46:08.726 [LogStash::Runner] runner - Using config.test_and_exit mode. Config Validation Result: OK. Exiting Logstash [root@logstash105.yinzhengjie.com ~]#
[root@logstash105.yinzhengjie.com ~]# systemctl restart logstash
4>.在redis验证数据
[root@redis104.yinzhengjie.com ~]# redis-cli -h 172.200.5.104 172.200.5.104:6379> AUTH yinzhengjie OK 172.200.5.104:6379> 172.200.5.104:6379> KEYS * 1) "filebeat-syslog-172-200-5-103" 2) "filebeat-nginx-172-200-5-103" 172.200.5.104:6379> 172.200.5.104:6379>
5>.再使用logstash将redis数据写入elaticseach集群
[root@es103.yinzhengjie.com ~]# vim /etc/logstash/conf.d/redis-to-elasticsearch.conf [root@es103.yinzhengjie.com ~]# [root@es103.yinzhengjie.com ~]# cat /etc/logstash/conf.d/redis-to-elasticsearch.conf input { redis { host => "redis104.yinzhengjie.com" port => "6379" password => "yinzhengjie" db => "0" key => "filebeat-syslog-172-200-5-103" data_type => "list" #codec => "json" } redis { host => "redis104.yinzhengjie.com" port => "6379" password => "yinzhengjie" db => "0" key => "filebeat-nginx-172-200-5-103" data_type => "list" codec => "json" } } output { if [fields][app] == "ubuntu-syslog" { elasticsearch { hosts => ["http://es101.yinzhengjie.com:9200","http://es102.yinzhengjie.com:9200"] index => "yinzhengjie-logstash-syslog-%{+YYYY.MM.dd}" } } if [fields][app] == "nginx" { elasticsearch { hosts => ["http://es101.yinzhengjie.com:9200","http://es102.yinzhengjie.com:9200"] index => "yinzhengjie-logstash-nginx-%{+YYYY.MM.dd}" } } } [root@es103.yinzhengjie.com ~]#
[root@es103.yinzhengjie.com ~]# systemctl restart logstash
