Logstash收集日志

1.Logstash的配置文件

[root@web01 ~]# vim /etc/logstash/logstash.yml
path.config: /etc/logstash/conf.d

2.logstash收集日志文件到文件

[root@web01 ~]# vim /etc/logstash/conf.d/file_file.conf
input {
  file {
    path => "/var/log/messages"
    start_position => "beginning"
  }
}
output {
  file {
    path => "/tmp/messages_%{+YYYY-MM-dd}.log"
  }
}

3.logstash收集日志文件到ES

[root@web01 ~]# vim /etc/logstash/conf.d/file_es.conf
input {
  file {
    path => "/var/log/messages"
    start_position => "beginning"
  }
}
output {
  elasticsearch {
    hosts => ["172.16.1.51:9200"]
    index => "messages_%{+YYYY-MM-dd}.log"
  }
}

4.Logstash收集多日志到文件

[root@web01 ~]# vim /etc/logstash/conf.d/file_file.conf
input {
  file {
    type => "messages_log"
    path => "/var/log/messages"
    start_position => "beginning"
  }
  file {
    type => "secure_log"
    path => "/var/log/secure"
    start_position => "beginning"
  }       
}        
output {  
  if [type] == "messages_log" { 
    file {
      path => "/tmp/messages_%{+YYYY-MM-dd}"
    }        
  }
  if [type] == "secure_log" {
    file {
      path => "/tmp/secure_%{+YYYY-MM-dd}"
    }
  } 
}

5.Logstash收集多日志到ES

1）方法一：

[root@web01 ~]# vim /etc/logstash/conf.d/more_es.conf 
input {
  file {
    type => "messages_log"
    path => "/var/log/messages"
    start_position => "beginning"
  }
  file {
    type => "secure_log"
    path => "/var/log/secure"
    start_position => "beginning"
  }
}
output {
  if [type] == "messages_log" {
    elasticsearch {
      hosts => ["10.0.0.51:9200"]
      index => "messages_%{+YYYY-MM-dd}"
    }
  }
  if [type] == "secure_log" {
    elasticsearch {
      hosts => ["10.0.0.51:9200"]
      index => "secure_%{+YYYY-MM-dd}"
    }
  }
}

[root@web01 ~]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/more_es.conf &

#启动后查看页面

2）方法二：

[root@web01 ~]# vim /etc/logstash/conf.d/more_es_2.conf 
input {
  file {
    type => "messages_log"
    path => "/var/log/messages"
    start_position => "beginning"
  }
  file {
    type => "secure_log"
    path => "/var/log/secure"
    start_position => "beginning"
  }
}
output {
  elasticsearch {
    hosts => ["10.0.0.51:9200"]
    index => "%{type}_%{+YYYY-MM-dd}"
  }
}

[root@web01 ~]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/more_es_2.conf --path.data=/data/logstash/more_es_2 &

3）启动多实例

#创建不同的数据目录
[root@web01 ~]# mkdir /data/logstash/more_es_2
[root@web01 ~]# mkdir /data/logstash/more_es

#启动时使用--path.data指定数据目录
[root@web01 ~]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/more_es.conf --path.data=/data/logstash/more_es &
[root@web01 ~]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/more_es_2.conf --path.data=/data/logstash/more_es_2 &

#如果资源充足，可以使用多实例收集多日志，如果服务器资源不足，启动不了多实例，配置一个文件收集多日志启动