zoukankan      html  css  js  c++  java
  • ELK集群搭建 --(二)

    #收集nginx访问日志

    #安装nginx

    root@web-1:/usr/local/src# wget https://nginx.org/download/nginx-1.18.0.tar.gz

    root@web-1:/usr/local/src# tar xvf nginx-1.18.0.tar.gz

    root@web-1:/usr/local/src#cd nginx-1.18.0/

    root@web-1:/usr/local/src#./configure --prefix=/apps/nginx

    root@web-1:/usr/local/src#make

    root@web-1:/usr/local/src#make install

    root@web-1:/etc/logstash/conf.d# vim /apps/nginx/conf/nginx.conf

    #access_log logs/access.log main;
    log_format access_json '{"@timestamp":"$time_iso8601",'
    '"host":"$server_addr",'
    '"clientip":"$remote_addr",'
    '"size":$body_bytes_sent,'
    '"responsetime":$request_time,'
    '"upstreamtime":"$upstream_response_time",'
    '"upstreamhost":"$upstream_addr",'
    '"http_host":"$host",' '"url":"$uri",'
    '"domain":"$host",'
    '"xff":"$http_x_forwarded_for",'
    '"referer":"$http_referer",'
    '"status":"$status"}';
    access_log logs/access.log access_json;

     #添加配置

    root@web-1:/etc/logstash/conf.d# vim /etc/logstash/conf.d/log-to-es.conf

     

     #重启logstash

    root@web-1:/etc/logstash/conf.d# systemctl restart logstash.service

    #通过 rsyslog 收集 haproxy 日志
    [root@haproxy-118 ~]# yum install haproxy
    [root@haproxy-118 ~]# vim /etc/haproxy/haproxy.cfg

    listen kibana
    bind 10.0.0.118:80
    mode http
    server kibana1 10.0.0.151:5601 check inter 2s fall 3 rise 5

    [root@haproxy-118 ~]# vim /etc/rsyslog.conf

    $ModLoad imudp
    $UDPServerRun 514

    local2.* @@10.0.0.154:2556

    [root@haproxy-118 ~]# systemctl restart rsyslog

    [root@haproxy-118 ~]# systemctl restart haproxy

    root@logstash1:/etc/logstash/conf.d# vim rsyslog.conf

    input {
    syslog {
    host => "10.0.0.154"
    port => "2556"
    type => "rsyslog"
    }
    }

    output {
    if [type] == "rsyslog" {
    elasticsearch {
    hosts => ["10.0.0.151:9200"]
    index => "songyk-rsyslog-%{+YYYY.MM.dd}"
    }
    }
    }
    root@logstash1:/etc/logstash/conf.d# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/rsyslog.conf  -t

    root@logstash1:/etc/logstash/conf.d# systemctl restart logstash.service 

     

    #logstash 收集日志并写入 redis 

    将web端的日志存入redis,logsstash从redis取出数据,logstash将数据通过haproxy发送给elasticsearch

    root@redis:~# apt install redis

    root@redis:~# vim /etc/redis/redis.conf

    requirepass 12345678

    save ""

    #save 900 1
    #save 300 10
    #save 60 10000

     root@logstash1:~# vim /etc/logstash/conf.d/songyk-redis-to-es.conf

    input {
    redis {
    data_type => "list"
    key => "nginx-acceslog"
    host => "10.0.0.155"
    port => "6379"
    db => "1"
    password => "12345678"
    }

    redis {
    data_type => "list"
    key => "nginx-errorlog"
    host => "10.0.0.155"
    port => "6379"
    db => "1"
    password => "12345678"

    }

    redis {
    data_type => "list"
    key => "tomcat-accesslog"
    host => "10.0.0.155"
    port => "6379"
    db => "0"
    password => "12345678"
    }

    redis {
    data_type => "list"
    key => "systemlog"
    host => "10.0.0.155"
    port => "6379"
    db => "0"
    password => "12345678"

    }


    }
    output {
    if [type] == "nginx-acceslog" {
    elasticsearch {
    hosts => ["10.0.0.118:9200"]
    index => "songyk-logstash-nginx-accesslog-%{+YYY.MM.dd}"
    }
    }

    if [type] == "nginx-errorlog" {
    elasticsearch {
    hosts => ["10.0.0.118:9200"]
    index => "songyk-logstash-nginx-errorlog-%{+YYY.MM.dd}"
    }
    }

    if [type] == "tomcat-acceslog" {
    elasticsearch {
    hosts => ["10.0.0.118:9200"]
    index => "songyk-logstash-tomcat-accesslog-%{+YYY.MM.dd}"
    }
    }

    if [type] == "systemlog" {
    elasticsearch {
    hosts => ["10.0.0.118:9200"]
    index => "songyk-logstash-systemlog-%{+YYY.MM.dd}"
    }
    }

    }

    root@web-1:/apps/apache-tomcat-8.5.57# cat /etc/logstash/conf.d/log-to-es.conf
    input {
    file {
    path => "/apps/apache-tomcat-8.5.57/logs/tomcat_access_log.*.log"
    type => "tomcat-acceslog"
    start_position => "beginning"
    stat_interval => "3"
    codec => json
    }

    file {
    path => "/var/log/syslog"
    type => "systemlog"
    start_position => "beginning"
    stat_interval => "3"
    }

    file {
    path => "/apps/nginx/logs/access.log"
    type => "nginx-acceslog"
    start_position => "beginning"
    stat_interval => "3"
    codec => json
    }

    file {
    path => "/apps/nginx/logs/error.log"
    type => "nginx-errorlog"
    start_position => "beginning"
    stat_interval => "3"
    }
    }

    output {
    if [type] == "tomcat-acceslog" {
    redis {
    data_type => "list"
    key => "tomcat-accesslog"
    host => "10.0.0.155"
    port => "6379"
    db => "0"
    password => "12345678"
    }
    }

    if [type] == "systemlog" {
    redis {
    data_type => "list"
    key => "systemlog"
    host => "10.0.0.155"
    port => "6379"
    db => "0"
    password => "12345678"

    }
    }
    if [type] == "nginx-acceslog" {
    redis {
    data_type => "list"
    key => "nginx-acceslog"
    host => "10.0.0.155"
    port => "6379"
    db => "1"
    password => "12345678"

    }
    }

    if [type] == "nginx-errorlog" {
    redis {
    data_type => "list"
    key => "nginx-errorlog"
    host => "10.0.0.155"
    port => "6379"
    db => "1"
    password => "12345678"
    }
    }

    }

    [root@haproxy-118 ~]# vim /etc/haproxy/haproxy.cfg 

    listen elasticsearch
    bind 10.0.0.118:9200
    mode tcp
    server es1 10.0.0.151:9200 check inter 2s fall 3 rise 5
    server es2 10.0.0.152:9200 check inter 2s fall 3 rise 5
    server es3 10.0.0.153:9200 check inter 2s fall 3 rise 5

     #安装metricbeat

     查看服务器指标

    root@web-1:/usr/local/src# dpkg -i metricbeat-7.12.1-amd64.deb

    root@web-1:/usr/local/src# vim /etc/metricbeat/metricbeat.yml

     未完待续。。。。。。

  • 相关阅读:
    lqb 基础练习 数列特征
    lqb 基础练习 查找整数 (遍历)
    lqb 基础练习 杨辉三角形
    lqb 基础练习 特殊的数字
    lqb 基础练习 回文数
    lqb 基础练习 特殊回文数
    lqb 基础练习 十进制转十六进制
    lqb 基础练习 十六进制转十进制
    lqb 基础练习 十六进制转八进制 (字符串进行进制转化)
    Git详细操作
  • 原文地址:https://www.cnblogs.com/syk-1994/p/14898232.html
Copyright © 2011-2022 走看看