zoukankan      html  css  js  c++  java
  • logstash收集nginx访问日志

    logstash收集nginx访问日志

    安装nginx

    #直接yum安装:
    [root@elk-node1 ~]# yum install nginx  -y
    官方文档:http://nginx.org/en/docs/http/ngx_http_log_module.html#log_format
    #修改配置文件的日志格式:
    vim /etc/nginx/nginx.conf  
    #在http模块中添加
              log_format json '{"@timestamp":"$time_iso8601",'
                               '"@version":"1",'
                               '"client":"$remote_addr",'
                               '"url":"$uri",'
                               '"status":"$status",'
                               '"domain":"$host",'
                               '"host":"$server_addr",'
                               '"size":$body_bytes_sent,'
                               '"responsetime":$request_time,'
                               '"referer": "$http_referer",'
                               '"ua": "$http_user_agent"'
                   '}';
    #在server模块中添加
    access_log  /var/log/nginx/access_json.log  json;
    #修改后的nginx.conf文件
    [root@elk-node1 ~]# grep -Ev "#|^&" /etc/nginx/nginx.conf
    
    user nginx;
    worker_processes auto;
    error_log /var/log/nginx/error.log;
    pid /run/nginx.pid;
    
    include /usr/share/nginx/modules/*.conf;
    
    events {
        worker_connections 1024;
    }
    
    http {
        log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                          '$status $body_bytes_sent "$http_referer" '
                          '"$http_user_agent" "$http_x_forwarded_for"';
              log_format json '{"@timestamp":"$time_iso8601",'
                               '"@version":"1",'
                               '"client":"$remote_addr",'
                               '"url":"$uri",'
                               '"status":"$status",'
                               '"domain":"$host",'
                               '"host":"$server_addr",'
                               '"size":$body_bytes_sent,'
                               '"responsetime":$request_time,'
                               '"referer": "$http_referer",'
                               '"ua": "$http_user_agent"'
                   '}';
        sendfile            on;
        tcp_nopush          on;
        tcp_nodelay         on;
        keepalive_timeout   65;
        types_hash_max_size 2048;
    
        include             /etc/nginx/mime.types;
        default_type        application/octet-stream;
    
        include /etc/nginx/conf.d/*.conf;
    
        server {
            listen       80 default_server;
            listen       [::]:80 default_server;
            server_name  _;
            root         /usr/share/nginx/html;
    
            include /etc/nginx/default.d/*.conf;
        access_log  /var/log/nginx/access_json.log  json;
    
            location / {
            }
    
            error_page 404 /404.html;
                location = /40x.html {
            }
    
            error_page 500 502 503 504 /50x.html;
                location = /50x.html {
            }
        }
    
    
    }
    #启动:
    [root@controller ~]# systemctl start nginx
    root@elk-node1 ~]# ss -lntp|grep 80
    LISTEN     0      511          *:80                       *:*                   users:(("nginx",pid=8045,fd=6),("nginx",pid=8044,fd=6),("nginx",pid=8043,fd=6))
    LISTEN     0      511         :::80                      :::*                   users:(("nginx",pid=8045,fd=7),("nginx",pid=8044,fd=7),("nginx",pid=8043,fd=7))
    

    浏览器访问:http://192.168.247.135/

    查看nginx日志

    编写logstash

    #添加nginx日志格式到之前logstash的elk-log.yml
    [root@elk-node1 ~]# cat /etc/logstash/conf.d/elk_log.conf
    input {
        file {
          path => "/var/log/messages"
          type => "system"
          start_position => "beginning"
        }
        file {
           path => "/var/log/elasticsearch/hejianlai.log"
           type => "es-error"
           start_position => "beginning"
          codec => multiline {
              pattern => "^["
              negate => true
              what => "previous"
            }
        }
           file {
           path => "/var/log/nginx/access_json.log"
           codec => json
           start_position => "beginning"
           type => "nginx-log"
        }
    }
    output {
        
        if [type] == "system"{
            elasticsearch {
               hosts => ["192.168.247.135:9200"]
               index => "systemlog-%{+YYYY.MM.dd}"
            }
        }
     
        if [type] == "es-error"{
            elasticsearch {
               hosts => ["192.168.247.135:9200"]
               index => "es-error-%{+YYYY.MM.dd}"
            }
        }
           if [type] == "nginx-log"{
            elasticsearch {
               hosts => ["192.168.247.135:9200"]
               index => "nginx-log-%{+YYYY.MM.dd}"
            }
        }
    }
    
    

     #添加--configtest参数检查配置语法是否有误!!!
    [root@elk-node1 ~]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/elk_log.conf --configtest
    Configuration OK
    #把之前后台运行的进程kill掉重启:
    [root@elk-node1 ~]# ps aux|grep elk
    root       3248  0.8  6.0 3632960 234924 pts/2  Sl   11:25   1:10 /usr/local/java/jdk1.8.0_171/bin/java -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -Djava.awt.headless=true -XX:CMSInitiatingOccupancyFraction=75 -XX:+UseCMSInitiatingOccupancyOnly -XX:+HeapDumpOnOutOfMemoryError -Xmx1g -Xss2048k -Djffi.boot.library.path=/opt/logstash/vendor/jruby/lib/jni -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -Djava.awt.headless=true -XX:CMSInitiatingOccupancyFraction=75 -XX:+UseCMSInitiatingOccupancyOnly -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/opt/logstash/heapdump.hprof -Xbootclasspath/a:/opt/logstash/vendor/jruby/lib/jruby.jar -classpath ::/usr/local/java/jdk1.8.0_171/lib:/usr/local/java/jdk1.8.0_171/jre/lib -Djruby.home=/opt/logstash/vendor/jruby -Djruby.lib=/opt/logstash/vendor/jruby/lib -Djruby.script=jruby -Djruby.shell=/bin/sh org.jruby.Main --1.9 /opt/logstash/lib/bootstrap/environment.rb logstash/runner.rb agent -f /etc/logstash/conf.d/elk_log.conf
    root       8135  0.0  0.0 112704   976 pts/0    S+   13:38   0:00 grep --color=auto elk
    [root@elk-node1 ~]# kill -9 3248
    You have new mail in /var/spool/mail/root
    [root@elk-node1 ~]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/elk_log.conf &
    [1] 8178

    kibana添加nginx日志

    首先在es插件中我们能看到nginx-log的索引

     设置kibana

  • 相关阅读:
    Populating Next Right Pointers in Each Node II
    Populating Next Right Pointers in Each Node
    Construct Binary Tree from Preorder and Inorder Traversal
    Construct Binary Tree from Inorder and Postorder Traversal
    Path Sum
    Symmetric Tree
    Solve Tree Problems Recursively
    632. Smallest Range(priority_queue)
    609. Find Duplicate File in System
    poj3159最短路spfa+邻接表
  • 原文地址:https://www.cnblogs.com/Dev0ps/p/9313418.html
Copyright © 2011-2022 走看看