收集TCP/UDP日志

收集TCP/UDP日志

通过logstash的tcp/udp插件收集日志，通常用于在向elasticsearch日志补录丢失的部分日志，可以将丢失的日志通过一个TCP端口直接写入到elasticsearch服务器。

1.配置Logstash

#进入Logstash配置文件目录
[root@redis01 ~]# cd /etc/logstash/conf.d/
#编辑Logstash配置文件
[root@redis01 conf.d]# vim tcp.conf
input {
  tcp {
    port => 1234
    type => "tcplog"
    mode => "server"
  }
}

output {
  stdout {
    codec => rubydebug
  }
}

2.启动

#启动Logstash
[root@redis01 conf.d]# /usr/share/logstash/bin/logstash -f  /etc/logstash/conf.d/tcp.conf
#检测端口是否启动成功
[root@redis01 ~]# netstat -lntup
tcp        0      0 :::1234                     :::*                        LISTEN      8656/java

3.使用telnet测试

[root@redis02 ~]# telnet 172.16.1.81 1234
Trying 172.16.1.81...
Connected to 172.16.1.81.
Escape character is '^]'.
13
12335346457thgdfhbd


#查看
{
          "port" => 58991,
      "@version" => "1",
    "@timestamp" => 2020-12-08T16:58:01.351Z,
          "host" => "172.16.1.82",
       "message" => "13
",
          "type" => "tcplog"
}
{
          "port" => 58991,
      "@version" => "1",
    "@timestamp" => 2020-12-08T16:58:27.160Z,
          "host" => "172.16.1.82",
       "message" => "12335346457thgdfhbd
",
          "type" => "tcplog"
}

4.使用nc工具

1）安装nc工具

#使用yum安装nc
[root@web01 ~]# yum install -y nc

2）使用测试

1.使用nc传输数据
[root@web01 ~]# echo "test nc" | nc 10.0.0.81 1234

2.收集文件日志
[root@web01 ~]# cat /etc/passwd | nc 10.0.0.81 1234

3.实时收集远端服务器的日志
[root@web01 ~]# tail -f /var/log/nginx/access.log | nc 10.0.0.81 1234

5.收集多个tcp日志到ES

1）配置

[root@redis01 ~]# cat /etc/logstash/conf.d/tcp_es.conf 
input {
  tcp {
    port => 1234
    type => "nginxlog"
    mode => "server"
  }
  tcp {
    port => "2345"
    type => "tomcatlog"
    mode => "server"
  }
}
output {
  if [type] == "nginxlog" {
    elasticsearch {
      hosts => ["10.0.0.71:9200"]
      index => "tcp_nginxlog_%{+YYYY-MM-dd}"
    }
  }
  if [type] == "tomcatlog" {
    elasticsearch {
      hosts => ["10.0.0.71:9200"]
      index => "tcp_tomcatlog_%{+YYYY-MM-dd}"
    }
  }
}

2）启动

[root@redis01 ~]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/tcp_es.conf

3）测试

[root@web01 ~]# tail -f /var/log/nginx/access.log | nc 10.0.0.81 1234

[root@web01 ~]# tail -f /usr/local/tomcat/logs/tomcat_access_json.$(date +%F).log | nc 10.0.0.81 2345

# 页面查看索引