日志分析 第五章 安装logstash

logstash是java应用，依赖JDK，首先需要安装JDK，在安装jdk过程中，logstash-2.3.4使用JDK-1.7版本有bug，使用JDK-1.8版本正常，因此我们安装JDK-1.8版本。

安装JDK

官网地址：http://www.oracle.com/technetwork/java/javase/downloads/jdk8-downloads-2133151.html

# rpm -ivh jdk-8u101-linux-x64.rpm
# echo "export JAVA_HOME=/usr/java/latest" >> /etc/profile
# echo "export PATH=$PATH:$JAVA_HOME/bin" >> /etc/profile
# source /etc/profile
# java -version
java version "1.8.0_101"
Java(TM) SE Runtime Environment (build 1.8.0_101-b13)
Java HotSpot(TM) 64-Bit Server VM (build 25.101-b13, mixed mode)

安装logstash

官网地址：https://www.elastic.co/products/logstash

# tar xf logstash-2.3.4.tar.gz -C /usr/local/app/
# ln -sv /usr/local/app/logstash-2.3.4 /usr/local/logstash
# cd /usr/local/logstash
# mkdir patterns

NGUSERNAME [a-zA-Z.@-+_%]+
NGUSER %{NGUSERNAME}
NGINXACCESS %{IPORHOST:server_name} %{IPORHOST:server_ip} %{IPORHOST:client_ip} %{NUMBER:client_port} [%{HTTPDATE:timestamp}] "%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}" %{NUMBER:response}  (?:%{NUMBER:body_bytes_sent}|-) (?:"(?:%{URI:referrer}|-)"|%{QS:referrer}) %{QS:agent} %{NUMBER:request_time} (?:%{NUMBER:upstream_response_time}|-)

SYSLOGBASE2 (?:%{SYSLOGTIMESTAMP:timestamp}|%{TIMESTAMP_ISO8601:timestamp8601}) (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}:
SYSLOGPAMSESSION %{SYSLOGBASE} (?=%{GREEDYDATA:message})%{WORD:pam_module}(%{DATA:pam_caller}): session %{WORD:pam_session_state} for user %{USERNAME:username}(?: by %{GREEDYDATA:pam_by})?

CRON_ACTION [A-Z ]+
CRONLOG %{SYSLOGBASE} (%{USER:user}) %{CRON_ACTION:action} (%{DATA:message})

SYSLOGLINE %{SYSLOGBASE2} %{GREEDYDATA:message}

# IETF 5424 syslog(8) format (see http://www.rfc-editor.org/info/rfc5424)
SYSLOG5424PRI <%{NONNEGINT:syslog5424_pri}>
SYSLOG5424SD [%{DATA}]+
SYSLOG5424BASE %{SYSLOG5424PRI}%{NONNEGINT:syslog5424_ver} +(?:%{TIMESTAMP_ISO8601:syslog5424_ts}|-) +(?:%{HOSTNAME:syslog5424_host}|-) +(?:%{WORD:syslog5424_app}|-) +(?:%{WORD:syslog5424_proc}|-) +(?:%{WORD:syslog5424_msgid}|-) +(?:%{SYSLOG5424SD:syslog5424_sd}|-|)

SYSLOG5424LINE %{SYSLOG5424BASE} +%{GREEDYDATA:syslog5424_msg}

编写配置文件

配置文件编写是一个难点，这里有一些示例供参考：https://github.com/logstash-plugins/logstash-patterns-core/tree/master/patterns

input {
  beats {
    port => 5048
    host => "10.80.2.181"
  }
}


filter {
  if [type] == "51-nginxaccesslog" {
  grok {
    patterns_dir => ["./patterns"]
    match => { "message" => "%{IPORHOST:server_name} %{IPORHOST:server_ip} %{IPORHOST:clientip} %{NUMBER:clientport} [%{HTTPDATE:timestamp}] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:status} (?:%{NUMBER:bytes}|-) %{QS:referrer} %{QS:agent} %{NUMBER:request_time:float} (?:%{NUMBER:upstream_time:float}|-)" } 
    remove_field => ["message"]
  }
    date {
      match => [ "timestamp", "dd/MMM/YYYY:HH:mm:ss Z"]
    }
  } else if [type] == "51-nginxerrorlog" {
  grok {
    patterns_dir => ["./patterns"]
    match => { "message" => "%{DATESTAMP} %{SYSLOG5424SD:nginx_error_level} %{GREEDYDATA:nginx_error_msg}"} 
    remove_field => ["message"]
  }
    date {
      match => [ "timestamp", "YYYY/MMM/dd HH:mm:ss"]
    }
  } else if [type] == "51-phperrorlog" {
  grok {
    patterns_dir => ["./patterns"]
    match => { "message" => "%{SYSLOG5424SD} (?:%{DATA:php_error_level}:) %{GREEDYDATA:error_msg}" }
    remove_field => ["message"]
  }
    date {
      match => [ "timestamp", "dd-MMM-YYYY HH:mm:ss Z"]
    }
  }
}


output {
  if "_grokparsefailure" in [tags] {
    file { path => "/var/log/logstash/grokparsefailure-%{[type]}-%{+YYYY.MM.dd}.log" }
  }
  elasticsearch {
    hosts => ["10.80.2.83:9200","10.80.2.84:9200"]
    sniffing => true
    manage_template => false
    template_overwrite => true
    index => "%{[type]}-%{+YYYY.MM.dd}"
    document_type => "%{[type]}"
  }
}

编写启动脚本

#!/bin/sh
# Init script for logstash
# Maintained by Elasticsearch
# Generated by pleaserun.
# Implemented based on LSB Core 3.1:
#   * Sections: 20.2, 20.3
#
### BEGIN INIT INFO
# Provides:          logstash
# Required-Start:    $remote_fs $syslog
# Required-Stop:     $remote_fs $syslog
# Default-Start:     2 3 4 5
# Default-Stop:      0 1 6
# Short-Description:
# Description:        Starts Logstash as a daemon.
### END INIT INFO

PATH=/sbin:/usr/sbin:/bin:/usr/bin
export PATH

if [ `id -u` -ne 0 ]; then
   echo "You need root privileges to run this script"
   exit 1
fi

name=logstash
pidfile="/usr/local/logstash/$name.pid"

LS_USER=nobody
LS_GROUP=nobody
LS_HOME=/usr/local/logstash
#LS_HOME=/home/logstash
LS_HEAP_SIZE="12g"
LS_LOG_DIR=/data/logstash/log
LS_LOG_FILE="${LS_LOG_DIR}/$name.log"
LS_CONF_DIR=/usr/local/logstash/conf.d
LS_OPEN_FILES=65535
LS_NICE=-20
LS_THREADS=8
KILL_ON_STOP_TIMEOUT=${KILL_ON_STOP_TIMEOUT-0} #default value is zero to this variable but could be updated by user request
LS_OPTS=""


[ -r /etc/default/$name ] && . /etc/default/$name
[ -r /etc/sysconfig/$name ] && . /etc/sysconfig/$name

program=/usr/local/logstash/bin/logstash
args="agent -f ${LS_CONF_DIR} -w ${LS_THREADS} -l ${LS_LOG_FILE} ${LS_OPTS}"

quiet() {
  "$@" > /dev/null 2>&1
  return $?
}

start() {

  LS_JAVA_OPTS="${LS_JAVA_OPTS} -Djava.io.tmpdir=${LS_HOME}"
  HOME=${LS_HOME}
  export PATH HOME LS_HEAP_SIZE LS_JAVA_OPTS LS_USE_GC_LOGGING LS_GC_LOG_FILE

  # chown doesn't grab the suplimental groups when setting the user:group - so we have to do it for it.
  # Boy, I hope we're root here.
  SGROUPS=$(id -Gn "$LS_USER" | tr " " "," | sed 's/,$//'; echo '')

  if [ ! -z $SGROUPS ]
  then
    EXTRA_GROUPS="--groups $SGROUPS"
  fi

  # set ulimit as (root, presumably) first, before we drop privileges
  ulimit -n ${LS_OPEN_FILES}

  # Run the program!
  nice -n ${LS_NICE} chroot --userspec $LS_USER:$LS_GROUP $EXTRA_GROUPS / sh -c "
    cd $LS_HOME
    ulimit -n ${LS_OPEN_FILES}
    exec "$program" $args
  " > "${LS_LOG_DIR}/$name.stdout" 2> "${LS_LOG_DIR}/$name.err" &

  # Generate the pidfile from here. If we instead made the forked process
  # generate it there will be a race condition between the pidfile writing
  # and a process possibly asking for status.
  echo $! > $pidfile

  echo "$name started."
  return 0
}

stop() {
  # Try a few times to kill TERM the program
  if status ; then
    pid=`cat "$pidfile"`
    echo "Killing $name (pid $pid) with SIGTERM"
    kill -TERM $pid
    # Wait for it to exit.
    for i in 1 2 3 4 5 6 7 8 9 ; do
      echo "Waiting $name (pid $pid) to die..."
      status || break
      sleep 1
    done
    if status ; then
      if [ $KILL_ON_STOP_TIMEOUT -eq 1 ] ; then
        echo "Timeout reached. Killing $name (pid $pid) with SIGKILL. This may result in data loss."
        kill -KILL $pid
        echo "$name killed with SIGKILL."
      else
        echo "$name stop failed; still running."
        return 1 # stop timed out and not forced
      fi
    else
      echo "$name stopped."
    fi
  fi
}

status() {
  if [ -f "$pidfile" ] ; then
    pid=`cat "$pidfile"`
    if kill -0 $pid > /dev/null 2> /dev/null ; then
      # process by this pid is running.
      # It may not be our pid, but that's what you get with just pidfiles.
      # TODO(sissel): Check if this process seems to be the same as the one we
      # expect. It'd be nice to use flock here, but flock uses fork, not exec,
      # so it makes it quite awkward to use in this case.
      return 0
    else
      return 2 # program is dead but pid file exists
    fi
  else
    return 3 # program is not running
  fi
}

reload() {
  if status ; then
    kill -HUP `cat "$pidfile"`
  fi
}

force_stop() {
  if status ; then
    stop
    status && kill -KILL `cat "$pidfile"`
  fi
}

configtest() {
  # Check if a config file exists
  if [ ! "$(ls -A ${LS_CONF_DIR}/* 2> /dev/null)" ]; then
    echo "There aren't any configuration files in ${LS_CONF_DIR}"
    return 1
  fi

  HOME=${LS_HOME}
  export PATH HOME

  test_args="--configtest -f ${LS_CONF_DIR} ${LS_OPTS}"
  $program ${test_args}
  [ $? -eq 0 ] && return 0
  # Program not configured
  return 6
}

case "$1" in
  start)
    status
    code=$?
    if [ $code -eq 0 ]; then
      echo "$name is already running"
    else
      start
      code=$?
    fi
    exit $code
    ;;
  stop) stop ;;
  force-stop) force_stop ;;
  status)
    status
    code=$?
    if [ $code -eq 0 ] ; then
      echo "$name is running"
    else
      echo "$name is not running"
    fi
    exit $code
    ;;
  reload) reload ;;
  restart)

    quiet configtest
    RET=$?
    if [ ${RET} -ne 0 ]; then
      echo "Configuration error. Not restarting. Re-run with configtest parameter for details"
      exit ${RET}
    fi
    stop && start
    ;;
  configtest)
    configtest
    exit $?
    ;;
  *)
    echo "Usage: $SCRIPTNAME {start|stop|force-stop|status|reload|restart|configtest}" >&2
    exit 3
  ;;
esac

exit $?

# chomd +x /etc/init.d/logstash
# chown –R nobody.nobody /usr/local/logstash
# chkconfig --add logstash