Linux系统 Elasticsearch、Logstash、Kibana集群部署文档 版本6.5.4

Elasticsearch、Logstash、Kibana集群部署文档 版本6.5.4

一、Elasticsearch部署文档

由于Elasticsearch、Logstash、Kibana均不能以root账号运行但是Linux对非root账号可并发操作的文件、线程都有限制。

1.修改文件限制修改系统文件

# vim /etc/security/limits.conf

* soft nofile 65536

* hard nofile 65536

* soft nproc 2048

* hard nproc 4096

2.调整进程数修改系统文件

# vim /etc/security/limits.d/20-nproc.conf

*          soft    nproc     4096

root       soft    nproc     unlimited

3.调整虚拟内存&最大并发连接修改系统文件

# im /etc/sysctl.conf

vm.max_map_count = 262144

fs.file-max = 655360

# sysctl -p

4.安装JDK并修改配置文件

# mkdir /usr/local/java

# tar -zxf jdk-8u201-linux-x64.tar.gz -C /usr/local/java/

# vim /etc/profile

JAVA_HOME=/usr/local/java/jdk1.8.0_201

CLASSPATH=.:$JAVA_HOME/lib/tools.jar:$JAVA_HOME/lib/dt.jar

PATH=$JAVA_HOME/bin:$HOME/bin:$HOME/.local/bin:$PATH

# source /etc/profile

5. 创建elasticsearch目录 并解压elasticsearch

# mkdir /usr/local/elasticsearch

# tar -zxf elasticsearch-6.5.4.tar.gz -C /usr/local/elasticsearch

# mkdir  /usr/local/elasticsearch/elasticsearch-6.5.4/data

查看配置文件

# grep '^[a-Z]' /usr/local/elasticsearch/elasticsearch-6.5.4/config/elasticsearch.yml

修改配置为文件主机名

# vim /etc/hosts

192.168.2.123 elasticsearch1

192.168.2.222 elasticsearch2

192.168.2.221 elasticsearch3

6.创建用户给予权限

# useradd elasticsearch

# chown -R elasticsearch.elasticsearch /usr/local/elasticsearch/

7.切换elasticsearch用户修改配置文件 启动elasticsearch服务

# su -  elasticsearch

# /usr/local/elasticsearch/elasticsearch-6.5.4/bin/elasticsearch -d

查看集群启动后能否正常工作

# curl -X GET "192.168.2.123:19200/"

查看当前索引和分片情况

# curl -i -XGET 'http://192.168.2.123:19200/_count?pretty'

使用head插件显示索引和分片情况安装elasticsearch-head插件

集群节点状态

注:status为green表示正常,yellow为警告,red为故障

# curl http://192.168.2.123:19200/_cluster/health?pretty

查看集群详细信息

# curl '192.168.2.123:19200/_cluster/state?pretty'

# http://192.168.2.123:19200/_cluster/state?pretty

#master配置文件

# ======================== Elasticsearch Configuration =========================
#判别节点是否是统一集群
cluster.name: elasticsearch-cluster
#节点的hostname
node.name: elasticsearch1
#该节点为主节点
node.master: true
#表示这不是数据节点
node.data: false
#本地ip端口
network.host: 192.168.2.123
http.port: 19200
#数据存放路径
path.data: /usr/local/elasticsearch/elasticsearch-6.5.4/data
#日志路径
path.logs: /usr/local/elasticsearch/elasticsearch-6.5.4/logs
#锁住内存，使内存不会再swap中使用
bootstrap.memory_lock: false
bootstrap.system_call_filter: false
http.cors.enabled: true
http.cors.allow-origin: "*"
#配置自动发现
discovery.zen.ping.unicast.hosts: ["192.168.2.222","192.168.2.123","192.168.2.221"]
discovery.zen.minimum_master_nodes: 1

#slave节点配置文件

# ======================== Elasticsearch Configuration =========================
cluster.name: elasticsearch-cluster
node.name: elasticsearch2
path.data: /usr/local/elasticsearch/elasticsearch-6.5.4/data
path.logs: /usr/local/elasticsearch/elasticsearch-6.5.4/logs
node.master: false
node.data: true
network.host: 192.168.2.222
http.port: 19200
bootstrap.memory_lock: false
bootstrap.system_call_filter: false
http.cors.enabled: true
http.cors.allow-origin: "*"
discovery.zen.minimum_master_nodes: 1
discovery.zen.ping.unicast.hosts: ["192.168.2.123","192.168.2.222","192.168.2.221"]

# ======================== Elasticsearch Configuration =========================
cluster.name: elasticsearch-cluster
node.name: elasticsearch3
path.data: /usr/local/elasticsearch/elasticsearch-6.5.4/data
path.logs: /usr/local/elasticsearch/elasticsearch-6.5.4/logs
node.master: false
node.data: true
network.host: 192.168.2.221
http.port: 19200
bootstrap.memory_lock: false
bootstrap.system_call_filter: false
http.cors.enabled: true
http.cors.allow-origin: "*"
discovery.zen.minimum_master_nodes: 1
discovery.zen.ping.unicast.hosts: ["192.168.2.123","192.168.2.222","192.168.2.221"]

二、elasticsearch-logstash部署文档

1、解压安装filebeat-6.5.4服务（日志收集器）

# tar -zxf filebeat-6.5.4-linux-x86_64.tar.gz -C /usr/local/elasticsearch/

# mv  /usr/local/elasticsearch/filebeat-6.5.4-linux-x86_64/ /usr/local/elasticsearch/filebeat-6.5.4

启动服务

# nohup ./usr/local/elasticsearch/filebeat-6.5.4/filebeat -c filebeat.yml &

查看配置文件

# grep "^s*[^# ].*$" /usr/local/elasticsearch/filebeat-6.5.4/filebeat.yml

2、解压安装logstash服务

#t ar -zxf logstash-6.5.4.tar.gz -C /usr/local/elasticsearch/

# ./logstash -f ../config/logstash.conf &

配置logstash服务配置文件

# ./logstash -t  -f /usr/local/elasticsearch/logstash-6.5.4/config/logstash.conf

logstash.conf配置文件

# Sample Logstash configuration for creating a simple
# Beats -> Logstash -> Elasticsearch pipeline.

#input {
# beats {
# port => 5044
# }
#}
input{
kafka{
bootstrap_servers => ["192.168.2.123:19092,192.168.2.222:19092"]
topics => "test"
codec => "json"
#client_id => "test"
#group_id => "test"
#auto_offset_reset => "latest" #从最新的偏移量开始消费
#consumer_threads => 5
#decorate_events => true #此属性会将当前topic、offset、group、partition等信息也带到message中
#topics => ["test","loge"] #数组类型，可配置多个topic
#type => "bhy" #所有插件通用属性,尤其在input里面配置多个数据源时很有用
}
}

output {
#{
#if [fields][tag] == "test"
#{
elasticsearch {
hosts => ["http://192.168.2.123:19200"]
index => "test-%{+YYYY-MM-dd}"
codec => "json"
#index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
#user => "elastic"
#password => "changeme"
}
}

三、elasticsearch-kibana部署文档

1.修改配置文件启动kibana服务

# tar -zxf kibana-6.5.4-linux-x86_64.tar.gz -C /usr/local/elasticsearch/

# mv /usr/local/elasticsearch/kibana-6.5.4-linux-x86_64 /usr/local/elasticsearch/kibana-6.5.4

# nohup /usr/local/elasticsearch/kibana-6.5.4/bin/kibana  > /dev/null 2>&1 &

# http://192.168.2.123:15601

配置文件

# Kibana is served by a back end server. This setting specifies the port to use.
server.port: 15601

# Specifies the address to which the Kibana server will bind. IP addresses and host names are both valid values.
# The default is 'localhost', which usually means remote machines will not be able to connect.
# To allow connections from remote users, set this parameter to a non-loopback address.
server.host: "192.168.2.123"

# Enables you to specify a path to mount Kibana at if you are running behind a proxy.
# Use the `server.rewriteBasePath` setting to tell Kibana if it should remove the basePath
# from requests it receives, and to prevent a deprecation warning at startup.
# This setting cannot end in a slash.
#server.basePath: ""

# Specifies whether Kibana should rewrite requests that are prefixed with
# `server.basePath` or require that they are rewritten by your reverse proxy.
# This setting was effectively always `false` before Kibana 6.3 and will
# default to `true` starting in Kibana 7.0.
#server.rewriteBasePath: false

# The maximum payload size in bytes for incoming server requests.
#server.maxPayloadBytes: 1048576

# The Kibana server's name. This is used for display purposes.
server.name: "192.168.2.123"

# The URL of the Elasticsearch instance to use for all your queries.
elasticsearch.url: "http://192.168.2.123:19200"

# When this setting's value is true Kibana uses the hostname specified in the server.host
# setting. When the value of this setting is false, Kibana uses the hostname of the host
# that connects to this Kibana instance.
#elasticsearch.preserveHost: true

# Kibana uses an index in Elasticsearch to store saved searches, visualizations and
# dashboards. Kibana creates a new index if the index doesn't already exist.
kibana.index: ".newkibana"

 

四、elasticsearch-head部署文档

安装配置docker服务

#yum -y install git docker

启动docker服务

#systemctl start docker

docker服务开机自动开启      

#systemctl enable docker

搜索elasticsearch-head容器镜像

#docker search elasticsearch-head           

下载elasticsearch-head基础容器镜像

# docker pull docker.io/alivv/elasticsearch-head

更改镜像名称

# docker tag docker.io/alivv/elasticsearch-head:latest elasticsearch-head:v1

查看下载的容器镜像

#docker images

部署elasticsearch-head服务

#docker run -d -p 19100:9100 --name elasticsearch-head --hostname elasticsearch-head --restart always elasticsearch-head:v1