自动化运维之日志系统Logstash解耦实践(八)

6.5消息队列解耦综合实践

1.将所有需要收集的日志写入一个配置文件,发送至node4的Redis服务(以下配置文件在各个节点上)。

 

1. [root@linux-node3 ~]# cat /etc/logstash/conf.d/input_file_output_redis.conf
2. input {
4. #system
5. syslog {
6. type => "system_rsyslog"
7. host => "192.168.90.203"
8. port => "514"
9. }
11. #java
12. file {
13. path => "/var/log/elasticsearch/xuliangwei.log"
14. type => "error_es"
15. start_position => "beginning"
16. codec => multiline {
17. pattern => "^["
18. negate => true
19. what => "previous"
20. }
21. }
23. #nginx
24. file {
25. path => "/var/log/nginx/access_json.log"
26. type => "access_nginx"
27. codec => "json"
28. start_position => "beginning"
29. }
30. }
33. output {
34. #多行文件判断
35. if [type] == "system_rsyslog" {
36. redis {
37. host => "192.168.90.204"
38. port=> "6379"
39. db => "6"
40. data_type => "list"
41. key => "system_rsyslog"
42. }
43. }
46. if [type] == "error_es" {
47. redis {
48. host => "192.168.90.204"
49. port=> "6379"
50. db => "6"
51. data_type => "list"
52. key => "error_es"
53. }
54. }
57. if [type] == "access_nginx" {
58. redis {
59. host => "192.168.90.204"
60. port=> "6379"
61. db => "6"
62. data_type => "list"
63. key => "access_nginx"
64. }
65. }
66. }

2.将Redis消息队列收集的所有日志，写入Elasticsearch集群。

 

1. [root@linux-node3 ~]# cat /etc/logstash/conf.d/input_redis_output_es.conf
2. input {
3. redis {
4. type => "system_rsyslog"
5. host => "192.168.90.204"
6. port=> "6379"
7. db => "6"
8. data_type => "list"
9. key => "system_rsyslog"
10. }
12. redis {
13. type => "error_es"
14. host => "192.168.90.204"
15. port=> "6379"
16. db => "6"
17. data_type => "list"
18. key => "error_es"
19. }
22. redis {
23. type => "access_nginx"
24. host => "192.168.90.204"
25. port=> "6379"
26. db => "6"
27. data_type => "list"
28. key => "access_nginx"
29. }
30. }
33. output {
34. #多行文件判断
35. if [type] == "system_rsyslog" {
36. elasticsearch {
37. hosts => ["192.168.90.201:9200","192.168.90.202:9200"]
38. index => "system_rsyslog_%{+YYYY.MM}"
39. }
40. }
42. if [type] == "error_es" {
43. elasticsearch {
44. hosts => ["192.168.90.201:9200","192.168.90.202:9200"]
45. index => "error_es_%{+YYYY.MM.dd}"
46. }
47. }
48. if [type] == "access_nginx" {
49. elasticsearch {
50. hosts => ["192.168.90.201:9200","192.168.90.202:9200"]
51. index => "access_nginx_%{+YYYY.MM.dd}"
52. }
53. }
54. }

3.查看Elasticsearch情况