zoukankan      html  css  js  c++  java
  • ELK5.0搭建部署

    ###关闭防火墙
    service iptables stop
    ###定义vi=vim
    alias vi=vim
    vi ~/.bashrc
    alias vi='vim'
    yum -y install lrzsz*
    创建文件夹
    mkdir -P /opt/soft
     
    上传一下包到soft路径
    172.25.11.77
    jdk-8u101-linux-x64.rpm
    kibana-5.0.0-x86_64.rpm
     
    172.25.11.78、79
    elasticsearch-5.0.0.rpm
    jdk-8u101-linux-x64.rpm
    logstash-5.0.0.rpm
     
    #####安装jdk
    rpm -ivh jdk-8u101-linux-x64.rpm
    java -version
     
    ######安装kibana(172.25.11.77)
    cd /opt/soft
    rpm -ivh kibana-5.0.0-x86_64.rpm
     
    ######安装elasticsearch、redis、logstash(172.25.11.78、172.25.11.79)
    cd /opt/soft
    rpm -ivh elasticsearch-5.0.0.rpm
    rpm -ivh logstash-5.0.0.rpm
    logstash启动脚本:
    /etc/init.d/logstash

    #!/bin/sh
    # Init script for logstash
    # Maintained by Elasticsearch
    # Generated by pleaserun.
    # Implemented based on LSB Core 3.1:
    # * Sections: 20.2, 20.3
    #
    ### BEGIN INIT INFO
    # Provides: logstash
    # Required-Start: $remote_fs $syslog
    # Required-Stop: $remote_fs $syslog
    # Default-Start: 2 3 4 5
    # Default-Stop: 0 1 6
    # Short-Description:
    # Description: Starts Logstash as a daemon.
    ### END INIT INFO

    PATH=/sbin:/usr/sbin:/bin:/usr/bin
    export PATH

    if [ `id -u` -ne 0 ]; then
    echo "You need root privileges to run this script"
    exit 1
    fi

    name=logstash
    pidfile="/var/run/$name.pid"

    LS_USER=root
    LS_GROUP=root
    LS_HOME=/var/lib/logstash
    LS_HEAP_SIZE="1000m"
    LS_LOG_DIR=/var/log/logstash
    LS_LOG_FILE="${LS_LOG_DIR}/$name.log"
    LS_CONF_DIR=/etc/logstash/conf.d/
    LS_OPEN_FILES=16384
    LS_NICE=19
    LS_OPTS=""


    [ -r /etc/default/$name ] && . /etc/default/$name
    [ -r /etc/sysconfig/$name ] && . /etc/sysconfig/$name

    program=/usr/share/logstash/bin/logstash
    args="-f ${LS_CONF_DIR} -l ${LS_LOG_FILE} ${LS_OPTS}"

    start() {

    LS_JAVA_OPTS="${LS_JAVA_OPTS} -Djava.io.tmpdir=${LS_HOME}"
    HOME=${LS_HOME}
    export PATH HOME LS_HEAP_SIZE LS_JAVA_OPTS LS_USE_GC_LOGGING

    # chown doesn't grab the suplimental groups when setting the user:group - so we have to do it for it.
    # Boy, I hope we're root here.
    SGROUPS=$(id -Gn "$LS_USER" | tr " " "," | sed 's/,$//'; echo '')

    if [ ! -z $SGROUPS ]
    then
    EXTRA_GROUPS="--groups $SGROUPS"
    fi

    # set ulimit as (root, presumably) first, before we drop privileges
    ulimit -n ${LS_OPEN_FILES}

    # Run the program!
    nice -n ${LS_NICE} chroot --userspec $LS_USER:$LS_GROUP $EXTRA_GROUPS / sh -c "
    cd $LS_HOME
    ulimit -n ${LS_OPEN_FILES}
    exec "$program" $args
    " > "${LS_LOG_DIR}/$name.stdout" 2> "${LS_LOG_DIR}/$name.err" &

    # Generate the pidfile from here. If we instead made the forked process
    # generate it there will be a race condition between the pidfile writing
    # and a process possibly asking for status.
    echo $! > $pidfile

    echo "$name started."
    return 0
    }

    stop() {
    # Try a few times to kill TERM the program
    if status ; then
    pid=`cat "$pidfile"`
    echo "Killing $name (pid $pid) with SIGTERM"
    kill -TERM $pid
    # Wait for it to exit.
    for i in 1 2 3 4 5 ; do
    echo "Waiting $name (pid $pid) to die..."
    status || break
    sleep 1
    done
    if status ; then
    if [ "$KILL_ON_STOP_TIMEOUT" -eq 1 ] ; then
    echo "Timeout reached. Killing $name (pid $pid) with SIGKILL. This may result in data loss."
    kill -KILL $pid
    echo "$name killed with SIGKILL."
    else
    echo "$name stop failed; still running."
    fi
    else
    echo "$name stopped."
    fi
    fi
    }

    status() {
    if [ -f "$pidfile" ] ; then
    pid=`cat "$pidfile"`
    if kill -0 $pid > /dev/null 2> /dev/null ; then
    # process by this pid is running.
    # It may not be our pid, but that's what you get with just pidfiles.
    # TODO(sissel): Check if this process seems to be the same as the one we
    # expect. It'd be nice to use flock here, but flock uses fork, not exec,
    # so it makes it quite awkward to use in this case.
    return 0
    else
    return 2 # program is dead but pid file exists
    fi
    else
    return 3 # program is not running
    fi
    }

    force_stop() {
    if status ; then
    stop
    status && kill -KILL `cat "$pidfile"`
    fi
    }


    case "$1" in
    start)
    status
    code=$?
    if [ $code -eq 0 ]; then
    echo "$name is already running"
    else
    start
    code=$?
    fi
    exit $code
    ;;
    stop) stop ;;
    force-stop) force_stop ;;
    status)
    status
    code=$?
    if [ $code -eq 0 ] ; then
    echo "$name is running"
    else
    echo "$name is not running"
    fi
    exit $code
    ;;
    restart)

    stop && start
    ;;
    *)
    echo "Usage: $SCRIPTNAME {start|stop|force-stop|status|restart}" >&2
    exit 3
    ;;
    esac

    exit $?

    ######安装redis
    # yum –y install tcl gcc  //安装测试依赖工具
    # tar xf redis-3.2.3.tar.gz  //解压至当前目录
    # cd redis-3.2.3  //切换至redis源码目录
    # make  //编译
    # make test  //有可能会失败,只是看看,不用在意
    # make install  //安装
    # mkdir /opt/redis/{db,conf} -pv  //创建redis安装目录
    # cp redis.conf /opt/redis/conf/  //复制配置文件至redis安装目录
    # cd src
    # cp redis-benchmark redis-check-aof redis-cli redis-server mkreleasehdr.sh /opt/redis/  //复制各文件至redis安装目录
    # ln -sv /opt/redis/redis-cli /usr/bin/  //将redis-cli可执行文件链接至/usr/bin目录,方便在任何目录下都可直接执行`redis-cli`
    # vim /opt/redis/conf/redis.conf  //修改redis.conf 中的 `daemonize`为`yes`,让server以守护进程在后台执行,这一步可以不做,因为后面要执行的脚本会自动创建这个文件,且这个值会设置为`yes`
    daemonize yes
    make install仅仅在你的系统上安装了二进制文件,不会替你默认配置init脚本和配置文件,为了把它用在生产环境而安装它,在源码目录的utils目录下Redis为系统提供了
    这样的一个脚本install_server.sh
    # cd /opt/soft/redis-3.2.3/utils
    # ./install_server.sh  //执行sh格式的安装脚本
    Welcome to the redis service installer
    This script will help you easily set up a running redis server
     
    Please select the redis port for this instance: [6379]
    Selecting default: 6379
    Please select the redis config file name [/etc/redis/6379.conf] /opt/redis/conf/redis.conf
    Please select the redis log file name [/var/log/redis_6379.log]
    Selected default - /var/log/redis_6379.log
    Please select the data directory for this instance [/var/lib/redis/6379] /opt/redis/db/6379.db
    Please select the redis executable path [/usr/local/bin/redis-server]          
    Selected config:
    Port           : 6379
    Config file    : /opt/redis/conf/redis.conf
    Log file       : /var/log/redis_6379.log
    Data dir       : /opt/redis/db/6379.db
    Executable     : /usr/local/bin/redis-server
    Cli Executable : /usr/local/bin/redis-cli
    Is this ok? Then press ENTER to go on or Ctrl-C to abort.
    Copied /tmp/6379.conf => /etc/init.d/redis_6379
    Installing service...
    Successfully added to chkconfig!
    Successfully added to runlevels 345!
    Starting Redis server...
    Installation successful!
     # chkconfig --add redis_6379  //将redis加入系统服务
    # chkconfig redis_6379 on  //加入开机启动
    # vim /opt/redis/conf/redis.conf
    requirepass Carsing2582#    //设置密码【坑1】:这步先不做,如果做了后面的测试连接redis的时候就需要认证)
    bind 0.0.0.0  //redis要打开远程访问允许,在 redis.conf 中 bind 这一行改成 0.0.0.0(运行任何机器连接,为了测试先这样)【此为坑2】
    # /etc/init.d/redis_6379 restart
    # ss -an| grep 6379          //端口已经被监听,证明服务已提供
    LISTEN     0      128               127.0.0.1:6379                     *:*    
    TIME-WAIT  0      0                 127.0.0.1:6379             127.0.0.1:34506
     
    ###修改 elasticsearch配置文件 存放所收集到的日志
    cd /etc/elasticsearch/
    vim elasticsearch.yml
    cluster.name: ELK ###集群名称
    node.name: node-1 ###节点编号
    node.master: true
    node.data: true
    path.data: /ELK/data ###elasticsearch数据路径
    path.logs: /ELK/logs ###elasticsearch日志路径
    network.host: elk02 ###本机主机名
    discovery.zen.ping.unicast.hosts: ["elk02", "elk03"] ###自动发现节点主机名
    discovery.zen.minimum_master_nodes: 2
    http.cors.enabled: true
    http.cors.allow-origin: "*"
    xpack.security.enabled: false
     
    启动异常:ERROR: bootstrap checks failed
    system call filters failed to install; check the logs and fix your configuration or disable system call filters at your own risk
    问题原因:因为Centos6不支持SecComp,而ES5.2.1默认bootstrap.system_call_filter为true进行检测,所以导致检测失败,失败后直接导致ES不能启动。详见 :https://github.com/elastic/elasticsearch/issues/22899
    解决方法:在elasticsearch.yml中配置bootstrap.system_call_filter为false,注意要在Memory下面:
    bootstrap.memory_lock: false
    bootstrap.system_call_filter: false
     
    设置线程池
    thread_pool:
    bulk:
    size: 5
    queue_size: 1000
    search:
    size: 5
    queue_size: 1000
    index:
    size: 5
    queue_size: 1000
     
    #xpack.monitoring.enabled: false
    #xpack.monitoring.exporters.my_local:
    # type: local
     
    indices.cache.filter.expire: 30d ###设置索引30天过期
    index.cache.filter: 30d
     
    vim /etc/elasticsearch/jvm.options ###内存酌情分配
    -Xms8g
    -Xmx8g
     
    vi /etc/security/limits.conf
    elasticsearch soft memlock unlimited
    elasticsearch hard memlock unlimited
    elasticsearch soft nofile 65536
    elasticsearch hard nofile 131072
    elasticsearch soft nproc 2048
    elasticsearch hard nproc 4096
    * soft nofile 65536
    * hard nofile 65536
    * soft memlock unlimited
    * hard memlock unlimited
     
     
     
     
    创建ELK数据、日志存放路径
    mkdir /ELK/data
    mkdir /ELK/logs
    chown -R elasticsearch:elasticsearch /ELK
    启动服务
    service elasticsearch start
    启动服务之后查看/ELK/logs路径中有日志,说明服务正常
    查看端口9200、9300
    9200端口是API 用来连接服务的接口
    9300是内部服务
     
    PS:报错
    max number of threads [1024] for user [lish] likely too low, increase to at least [2048]
    解决:切换到root用户,进入limits.d目录下修改配置文件。
    vi /etc/security/limits.d/90-nproc.conf 
    修改如下内容:
    * soft nproc 1024
    #修改为
    * soft nproc 2048
     
    ##修改主机名,定义dns
    vim /etc/sysconfig/network
    NETWORKING=yes
    HOSTNAME=elk02
    GATEWAY=172.25.0.254
     
    vim /etc/hosts
    127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
    ::1 localhost localhost.localdomain localhost6 localhost6.localdomain6
    172.25.11.77 elk01
    172.25.11.78 elk02
    172.25.11.79 elk03
    注:重启生效 reboot
     
    ###配置kibana
    cd /etc/kibana/
    vim kibana.yml
    server.port: 5601 ##定义kibana端口
    server.host: "172.25.11.77" ##服务主机IP
    elasticsearch.url: "http://172.25.11.78:9200" ##url调取elasticsearch
    elasticsearch.url: "http://172.25.11.79:9200" ##url调取elasticsearch
    ######启动kibana
    service kibana start
    查看端口是否存在
    netstat -anp | grep 5601
    tcp 0 0 172.25.11.77:5601 0.0.0.0:* LISTEN 8492/node
    ###web页面上访问
     
    ####安装elasticsearch-analysis-ik-5.0.0.zip 分词器
     
    ####安装elasticsearch-head插件
    由于elasticsearch5.0版本变化较大,目前elasticsearch5.0 暂时不支持直接安装,但是head作者提供了另一种安装方法
    ###安装elasticsearch-head
    cd /usr/share/elasticsearch
    git clone git://github.com/mobz/elasticsearch-head.git
    chmod -R 777 elasticsearch-head
    cd elasticsearch-head/
    PS:先修改license
    vim package.json
    "license": "Apache-2.0",
     
    npm install --force ###安装完node就会有npm命令
    cd /usr/share/elasticsearch/elasticsearch-head/node_modules/grunt/bin
    ./grunt server > /dev/null 2>&1 &
     
    ###安装node
    node-v6.10.1-linux-x64.tar
    解压
    tar -xvf node-v6.10.1-linux-x64.tar
    mv node-v6.10.1-linux-x64 /usr/share/elasticsearch/
    vim /etc/profile
    export NODE_HOME=/usr/share/elasticsearch/node-v6.10.1-linux-x64
    export PATH=$PATH:$NODE_HOME/bin
    source /etc/profile
    cd /usr/share/elasticsearch/node-v6.10.1-linux-x64/bin/
    ./node -v
    v6.10.1
    npm -v
    3.10.10
    ###安装grunt
    ######cd /usr/share/elasticsearch/elasticsearch-head
    ######npm install grunt-cli
    ###### npm install -g grunt --registry=https://registry.npm.taobao.org
    grunt --version
    #### 修改head目录下的Gruntfile.js配置,head默认监听127.0.0.1
    vim Gruntfile.js
    hostname: '*',
    connect: {
    server: {
    options: {
    port: 9100,
    hostname: '*', ####修改此处
    base: '.',
    keepalive: true
    }
    }
    cd _site/
    vim app.js
    this.base_uri = this.config.base_uri || this.prefs.get("app-base_uri") || "http://172.25.11.78:9200";
     
    ####elasticsearch配置允许跨域访问
    修改elasticsearch配置文件elasticsearch.yml
    cd /etc/elasticsearch/
    vim elasticsearch.yml
    http.cors.enabled: true
    http.cors.allow-origin: "*"
     
    ####重启elasticsearch,并启动node
    service elasticsearch restart ###启动elasticsearch
    cd /usr/share/elasticsearch/elasticsearch-head/ ###启动node
    ①#######node_modules/grunt/bin/grunt server > /dev/null 2>&1 &
     
    ②#######/usr/share/elasticsearch/elasticsearch-head/node_modules/grunt/bin/grunt server > /dev/null 2>&1 &
     
    查看集群信息
    curl '172.25.11.78:9200 '
    ip heap.percent ram.percent cpu load_1m load_5m load_15m node.role master name
    172.25.11.78 13 33 0 0.01 0.01 0.00 mdi - node-1
    172.25.11.79 10 33 0 0.00 0.02 0.00 mdi * node-2
     
    #####安装x-pack(marvel) 进行性能监控
    配置dns
    vim /etc/resolv.conf
    nameserver 210.73.88.1
     
    kibana上安装server端
    cd /usr/share/kibana/
    bin/kibana-plugin install x-pack
     
    es上安装agent端
    cd /usr/share/elasticsearch/
    bin/elasticsearch-plugin install x-pack
     
    卸载
    bin/elasticsearch-plugin remove x-pack
     
    PS:
    安装完需要重启kibana和es 默认用户名密码elastic/changeme
     
    查看license时间 默认一个月
    curl -XGET -u elastic:1qaz2wsx 'http://172.25.11.78:9200/_xpack/license'
    替换license 默认密码changeme
    curl -XPUT -u elastic 'http://172.25.11.78:9200/_xpack/license?acknowledge=true' -H "Content-Type: application/json" -d @zhaishaomin-fa370f7b-6eb7-4730-9b73-89b3d93471e3-v5.json
     
    由于是基本的license,所以需要禁用安全性
    vim /etc/kibana/kibana.yml
    vim /etc/elasticsearch/elasticsearch.yml
    xpack.security.enabled: false ####false为禁用
     
    ####安装logstash
    rpm -ivh logstash-5.0.0.rpm
    ############################################################################
    PS:安装出现一下错误:
    warning: rpmts_HdrFromFdno: Header V3 RSA/SHA256 Signature, key ID fd431d51: NOKEY 
    解决:
    rpm --import /etc/pki/rpm-gpg/RPM*
    ############################################################################
    cd /usr/share/logstash/
    mkdir config
    cp /etc/logstash/logstash.yml /usr/share/logstash/config/
    cp /etc/logstash/log4j2.properties /usr/share/logstash/config/
    ######cd /etc/logstash/conf.d#######
    vim input.conf
    input{
    #stdin{}
    file{
    path => "/opt/soft/sub-test.cm-inv.com_access_80.log"
    start_position => "beginning" ####end(从末尾读)beginning(从头开始读)
    stat_interval => 1
    type => "access_log"
    tags=> ["acc"]
    }
     
    }
     
    filter{
    grok{
    match => [ "message" , "[%{HTTPDATE:time_local}] "%{IP:client_ip}" "%{WORD:verb} (%{NOTSPACE:request}|-) (HTTP/%{NUMBER:http_version}|-)" "(%{GREEDYDATA:http_cookie}|-)" "(%{WORD:http_x_forword_for}|-)" (%{GREEDYDATA:nomean}|-) (?<user>[a-zA-Z._-]+|-)(?<status>[0-9._-]+|-) (?:%{NUMBER:body_bytes_sent:int}|-) (%{BASE16FLOAT:request_time:float}|-) "(%{GREEDYDATA:http_did}|-)" "(%{GREEDYDATA:http_x_up_calling_line_id}|-)" "(%{NOTSPACE:http_referrer}|-)" "%{GREEDYDATA:http_user_agent}" (%{BASE16FLOAT}|-)(%{NUMBER:content_length}|-)(%{GREEDYDATA:upstream_addr}|-)"]
    remove_field => [ "message","host","@version" ]
    }
    date{
    match=>["timestamp","dd/MMM/yyyy:HH:mm:ss Z"]
    target=>"@timestamp"
    }
    geoip{
    source => "client_ip"
    }
    }
    ###########
    output{
    elasticsearch {
    hosts => "192.168.125.129:9200"
    index => "logstash-%{type}-%{+YYYY.MM.dd}"
    }
    stdout {codec=> rubydebug }
    }
    ################
    /usr/share/logstash/bin/logstash -f input.conf
     
     
    bin/logstash  -e  ‘input { stdin {} } output { stdout {codec=>rubydebug} }’
     
     
    删除一个月的索引脚本
    #!/bin/bash
    data=`date -d '-1 months' +%Y-%m-%d`
    curl -XDELETE 'http://172.25.11.77:9200/*-'${data}''
     
     
    ELK学习10_ELK系列--实时日志分析系统ELK 部署与运行中的问题汇总 - wang_zhenwei的博客 - CSDN博客 http://blog.csdn.net/wang_zhenwei/article/details/50372000
     
  • 相关阅读:
    http编程中的get和post混合使用方式
    SQLServer实现作业依赖(非步骤)
    SQLServer实现两个库的字段长度自动更新
    Python+SQLite数据库实现服务端高并发写入
    sqlite数据库相关使用
    sqlite语法
    VBA关键字总结
    VS2005 .net2.0 TreeView.设置SelectedNodeStyle控制TreeView中选定节点的外观的
    SQLSERVER 2005 如何给sa用户设置空密码?
    解决超过远程连接数而无法连接服务器的问题 踢出已断开用户
  • 原文地址:https://www.cnblogs.com/zhaishaomin/p/7417233.html
Copyright © 2011-2022 走看看