Logstash收集Nginx访问日志and错误日志

一 将nginx日志转换为json格式

 31 #    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
 32 #                      '$status $body_bytes_sent "$http_referer" '
 33 #                      '"$http_user_agent" "$http_x_forwarded_for"';
 34 
 35 #    access_log  /var/log/nginx/access.log  main;
 36 
 37 log_format access_json '{"@timestamp":"$time_iso8601",'
 38         '"host":"$server_addr",'
 39         '"clientip":"$remote_addr",'
 40         '"size":$body_bytes_sent,'
 41         '"responsetime":$request_time,'
 42         '"upstreamtime":"$upstream_response_time",'
 43         '"upstreamhost":"$upstream_addr",'
 44         '"http_host":"$host",'
 45         '"url":"$uri",'
 46         '"domain":"$host",'
 47         '"xff":"$http_x_forwarded_for",'
 48         '"referer":"$http_referer",'
 49         '"status":"$status"}';
 50 access_log  /var/log/nginx/access_json.log  access_json;

  设置其他域名日志路径

 [root@localhost logs]# vim /etc/nginx/conf.d/test.p.com.conf 
  1 server {
  2         listen       80;
  3         server_name test.p.com;
  4         charset utf-8;
  5         #rewrite_log on;
  6         #error_log /var/log/nginx/carwww_error.log;
  7         access_log  /opt/vhosts/test/logs/access_json.log  access_json;
  8         root /opt/vhosts/test/www;

检查并重启Nginx

[root@localhost ~]# nginx -t
nginx: the configuration file /etc/nginx/nginx.conf syntax is ok
nginx: configuration file /etc/nginx/nginx.conf test is successful
[root@localhost ~]# systemctl restart nginx

检查Nginx日志格式

在线验证json格式网址：http://www.bejson.com/

[root@localhost ~]# tail -f /opt/vhosts/test/logs/access_json.log 
{"@timestamp":"2018-08-06T12:56:51+08:00","host":"192.168.10.172","clientip":"192.168.10.81","size":475,"responsetime":2.657,"upstreamtime":"2.657","upstreamhost":"127.0.0.1:9000","http_host":"test.p.com","url":"/index.php","domain":"test.p.com","xff":"-","referer":"-","status":"200"}

二 配置logstash收集Nginx和系统日志

[root@localhost ~]# cat /etc/logstash/conf.d/nginx.conf 
input {
      file {
          path => "/opt/vhosts/fatai/logs/access_json.log"
              start_position => "beginning"
          type => "nginx-accesslog-test"
          codec => json
                  stat_interval => "2"          
      }
      file {
           path => "/var/log/messages" 
               start_position => "beginning" 
           type => "systemlog-test"   
                   stat_interval => "2"          
      }
      file {
           type => "nginx-error-test" 
           path => [ "/var/log/nginx/error.log" ]
           tags => [ "nginx","error"]
               start_position => "beginning"
                   stat_interval => "2"          
       }
}


output {
    if [type] == "nginx-accesslog-test" {
      elasticsearch {
          hosts => ["192.168.10.10:9200"]
              index => "logstash-nginx-accesslog-test-%{+YYYY.MM.dd}"
      }}
    if [type] == "systemlog-test" {
      elasticsearch {
           hosts => ["192.168.10.10:9200"]
           index => "logstash-systemlog-test-%{+YYYY.MM.dd}"
      }}
    if [type] == "nginx-error-test" {
      elasticsearch {
          hosts => ["192.168.10.10:9200"]
          index => "logstash-nginx-error-test-%{+YYYY.MM.dd}"                                                                      
    }}
}

检查Logstash配置文件并重启

[root@localhost ~]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/nginx.conf -t
WARNING: Could not find logstash.yml which is typically located in $LS_HOME/config or /etc/logstash. You can specify the path using --path.settings. Continuing using the defaults
Could not find log4j2 configuration at path /usr/share/logstash/config/log4j2.properties. Using default config which logs errors to the console
Configuration OK
[root@localhost ~]# systemctl restart logstash.service

在Elasticsearch-head验证

三 加入到kibana

   本次kibana用的是5.6