整体策略：
将集群配置好：通过命令 curl ip:9200/_cluster/health?pretty 检查状态为green
然后再次修改配置文件启用xpack认证，并拷贝认证文件到各节点，重启集群的节点，访问api通过账号密码验证
1.配置基础环境
# 创建相关的日志和数据目录
mkdir -p /data/es/data
mkdir -p /data/es/logs
mkdir -p /data/esback
# 添加es程序运行的专用账户
groupadd -g 1500 elasticsearch
useradd -u 1500 -g elasticsearch elasticsearch
# 关闭swap分区
swapoff -a
# 配置内核参数
echo "fs.file-max = 1000000" >> /etc/sysctl.conf
echo "vm.max_map_count=262144" >> /etc/sysctl.conf
echo "vm.swappiness = 1" >> /etc/sysctl.conf
sysctl -p
sed -i 's/* soft nofile 65535/* soft nofile 655350/g' /etc/security/limits.conf
sed -i 's/* hard nofile 65535/* hard nofile 655350/g' /etc/security/limits.conf
sed -i 's#* soft nproc 4096##' /etc/security/limits.d/20-nproc.conf
# 最终效果
# vim /etc/security/limits.conf
root soft nofile 655350
root hard nofile 655350
* soft nofile 6553500
* hard nofile 6553500
elasticsearch soft memlock unlimited
elasticsearch hard memlock unlimited
# 注释文件/etc/pam.d/su中的
session include system-auth 一行即可。
# 修改/etc/security/limits.d/20-nproc.conf，新增
* soft memlock unlimited
* hard memlock unlimited
chown -R elasticsearch.elasticsearch /data/es
chown -R elasticsearch.elasticsearch /data/esback
chown -R elasticsearch.elasticsearch /usr/local/elk/elasticsearch-7.5.1
2.下载es程序
mkdir /usr/local/elk
cd /usr/local/elk
wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.4.2-linux-x86_64.tar.gz
tar xf elasticsearch-7.4.2-linux-x86_64.tar.gz
# 新版本的elasticsearch使用自带的openjdk13
# vim bin/profile
export JAVA_HOME=/usr/local/elk/elasticsearch-7.4.2/jdk
export PATH=$JAVA_HOME/bin:$PATH
#修改 /usr/local/elk/elasticsearch-7.4.2/config/elasticsearch.yml
# 一般为物理内存的一半，可以适当调整
-Xms4g
-Xmx4g
# 修改各自hosts文件
# online_image_es_pro
172.30.0.106 eus-image-design-es-kafka01
172.30.0.107 eus-image-design-es-kafka02
172.30.0.110 eus-image-design-mongo03
chown -R elasticsearch.elasticsearch /data/es
chown -R elasticsearch.elasticsearch /data/esback
chown -R elasticsearch.elasticsearch /usr/local/elk/elasticsearch-7.4.2
cd /usr/local/elk/elasticsearch-7.4.2
su elasticsearch
3.配置es
# 如下配置，启动elasticserch集群，不启用xpach认证
# 修改主节点参数
# cat /usr/local/elk/elasticsearch-7.4.2/config/elasticsearch.yml
cluster.name: online_image_es_pro
node.name: eus-image-design-es-kafka01
path.data: /data/es/data
path.logs: /data/es/logs
path.repo: ["/data/esback"]
bootstrap.memory_lock: true
bootstrap.system_call_filter: false
network.host: 172.30.0.106
http.port: 9200
transport.tcp.port: 9300
node.master: true
node.data: true
discovery.seed_hosts: ["172.30.0.106:9300", "172.30.0.107:9300", "172.30.0.110:9300"]
cluster.initial_master_nodes: ["172.30.0.106"]
discovery.zen.minimum_master_nodes: 1
gateway.recover_after_nodes: 1
transport.tcp.compress: true
http.cors.enabled: true
http.cors.allow-origin: "*"
# 以下配置可以减少当es节点短时间宕机或重启时shards重新分布带来的磁盘io读写浪费
discovery.zen.fd.ping_timeout: 300s
discovery.zen.fd.ping_retries: 8
discovery.zen.fd.ping_interval: 30s
discovery.zen.ping_timeout: 180s
# 密码部分的配置最好等集群配置没有问题后再进行
# xpack.security.enabled: true
# xpack.security.transport.ssl.enabled: true
# xpack.security.transport.ssl.verification_mode: certificate
# xpack.security.transport.ssl.keystore.path: /usr/local/elk/elasticsearch-7.4.2/config/elastic-certificates.p12
# xpack.security.transport.ssl.truststore.path: /usr/local/elk/elasticsearch-7.4.2/config/elastic-certificates.p12
# 数据节点配置
# cat /usr/local/elk/elasticsearch-7.4.2/config/elasticsearch.yml
cluster.name: online_image_es_pro
node.name: eus-image-design-es-kafka02
path.data: /data/es/data
path.logs: /data/es/logs
path.repo: ["/data/esback"]
bootstrap.memory_lock: true
bootstrap.system_call_filter: false
network.host: 172.30.0.107
http.port: 9200
transport.tcp.port: 9300
node.master: false
node.data: true
discovery.seed_hosts: ["172.30.0.106:9300", "172.30.0.107:9300", "172.30.0.110:9300"]
cluster.initial_master_nodes: ["172.30.0.106"]
discovery.zen.minimum_master_nodes: 1
gateway.recover_after_nodes: 1
transport.tcp.compress: true
http.cors.enabled: true
http.cors.allow-origin: "*"
# 以下配置可以减少当es节点短时间宕机或重启时shards重新分布带来的磁盘io读写浪费
discovery.zen.fd.ping_timeout: 300s
discovery.zen.fd.ping_retries: 8
discovery.zen.fd.ping_interval: 30s
discovery.zen.ping_timeout: 180s
# 密码部分的配置最好等集群配置没有问题后再进行
# xpack.security.enabled: true
# xpack.security.transport.ssl.enabled: true
# xpack.security.transport.ssl.verification_mode: certificate
# xpack.security.transport.ssl.keystore.path: /usr/local/elk/elasticsearch-7.4.2/config/elastic-certificates.p12
# xpack.security.transport.ssl.truststore.path: /usr/local/elk/elasticsearch-7.4.2/config/elastic-certificates.p12
# 数据节点配置
# cat /usr/local/elk/elasticsearch-7.4.2/config/elasticsearch.yml
cluster.name: online_image_es_pro
node.name: eus-image-design-mongo03
path.data: /data/es/data
path.logs: /data/es/logs
path.repo: ["/data/esback"]
bootstrap.memory_lock: true
bootstrap.system_call_filter: false
network.host: 172.30.0.110
http.port: 9200
transport.tcp.port: 9300
node.master: false
node.data: true
discovery.seed_hosts: ["172.30.0.106:9300", "172.30.0.107:9300", "172.30.0.110:9300"]
cluster.initial_master_nodes: ["172.30.0.106"]
discovery.zen.minimum_master_nodes: 1
gateway.recover_after_nodes: 1
transport.tcp.compress: true
http.cors.enabled: true
http.cors.allow-origin: "*"
# 以下配置可以减少当es节点短时间宕机或重启时shards重新分布带来的磁盘io读写浪费
discovery.zen.fd.ping_timeout: 300s
discovery.zen.fd.ping_retries: 8
discovery.zen.fd.ping_interval: 30s
discovery.zen.ping_timeout: 180s
# 密码部分的配置最好等集群配置没有问题后再进行
# xpack.security.enabled: true
# xpack.security.transport.ssl.enabled: true
# xpack.security.transport.ssl.verification_mode: certificate
# xpack.security.transport.ssl.keystore.path: /usr/local/elk/elasticsearch-7.4.2/config/elastic-certificates.p12
# xpack.security.transport.ssl.truststore.path: /usr/local/elk/elasticsearch-7.4.2/config/elastic-certificates.p12
# 确认集群状态ok
# curl 172.30.0.106:9200/_cluster/health?pretty
{
"cluster_name" : "online_image_es_pro",
"status" : "green",
"timed_out" : false,
"number_of_nodes" : 3,
"number_of_data_nodes" : 3,
"active_primary_shards" : 0,
"active_shards" : 0,
"relocating_shards" : 0,
"initializing_shards" : 0,
"unassigned_shards" : 0,
"delayed_unassigned_shards" : 0,
"number_of_pending_tasks" : 0,
"number_of_in_flight_fetch" : 0,
"task_max_waiting_in_queue_millis" : 0,
"active_shards_percent_as_number" : 100.0
}
# curl 172.30.0.106:9200/_cat/nodes
172.30.0.106 9 21 1 0.01 0.06 0.05 dilm * eus-image-design-es-kafka01
172.30.0.107 13 20 1 0.01 0.09 0.10 dil - eus-image-design-es-kafka02
172.30.0.110 13 41 1 0.02 0.09 0.07 dil - eus-image-design-mongo03
4.配置xpack密码认证
生成证书
[root@eus-image-design-es-kafka01:~]# cd /usr/local/elk/elasticsearch-7.4.2
[root@eus-image-design-es-kafka01:/usr/local/elk/elasticsearch-7.4.2]# su elasticsearch
[elasticsearch@eus-image-design-es-kafka01:/usr/local/elk/elasticsearch-7.4.2]$ bin/elasticsearch-certutil ca
WARNING: An illegal reflective access operation has occurred
WARNING: Illegal reflective access by org.bouncycastle.jcajce.provider.drbg.DRBG (file:/usr/local/elk/elasticsearch-7.4.2/lib/tools/security-cli/bcprov-jdk15on-1.61.jar) to constructor sun.security.provider.Sun()
WARNING: Please consider reporting this to the maintainers of org.bouncycastle.jcajce.provider.drbg.DRBG
WARNING: Use --illegal-access=warn to enable warnings of further illegal reflective access operations
WARNING: All illegal access operations will be denied in a future release
This tool assists you in the generation of X.509 certificates and certificate
signing requests for use with SSL/TLS in the Elastic stack.
The 'ca' mode generates a new 'certificate authority'
This will create a new X.509 certificate and private key that can be used
to sign certificate when running in 'cert' mode.
Use the 'ca-dn' option if you wish to configure the 'distinguished name'
of the certificate authority
By default the 'ca' mode produces a single PKCS#12 output file which holds:
* The CA certificate
* The CA's private key
If you elect to generate PEM format certificates (the -pem option), then the output will
be a zip file containing individual files for the CA certificate and private key
Please enter the desired output file [elastic-stack-ca.p12]:
Enter password for elastic-stack-ca.p12 :
[elasticsearch@eus-image-design-es-kafka01:/usr/local/elk/elasticsearch-7.4.2]$ bin/elasticsearch-certutil cert --ca elastic-stack-ca.p12
WARNING: An illegal reflective access operation has occurred
WARNING: Illegal reflective access by org.bouncycastle.jcajce.provider.drbg.DRBG (file:/usr/local/elk/elasticsearch-7.4.2/lib/tools/security-cli/bcprov-jdk15on-1.61.jar) to constructor sun.security.provider.Sun()
WARNING: Please consider reporting this to the maintainers of org.bouncycastle.jcajce.provider.drbg.DRBG
WARNING: Use --illegal-access=warn to enable warnings of further illegal reflective access operations
WARNING: All illegal access operations will be denied in a future release
This tool assists you in the generation of X.509 certificates and certificate
signing requests for use with SSL/TLS in the Elastic stack.
The 'cert' mode generates X.509 certificate and private keys.
* By default, this generates a single certificate and key for use
on a single instance.
* The '-multiple' option will prompt you to enter details for multiple
instances and will generate a certificate and key for each one
* The '-in' option allows for the certificate generation to be automated by describing
the details of each instance in a YAML file
* An instance is any piece of the Elastic Stack that requires an SSL certificate.
Depending on your configuration, Elasticsearch, Logstash, Kibana, and Beats
may all require a certificate and private key.
* The minimum required value for each instance is a name. This can simply be the
hostname, which will be used as the Common Name of the certificate. A full
distinguished name may also be used.
* A filename value may be required for each instance. This is necessary when the
name would result in an invalid file or directory name. The name provided here
is used as the directory name (within the zip) and the prefix for the key and
certificate files. The filename is required if you are prompted and the name
is not displayed in the prompt.
* IP addresses and DNS names are optional. Multiple values can be specified as a
comma separated string. If no IP addresses or DNS names are provided, you may
disable hostname verification in your SSL configuration.
* All certificates generated by this tool will be signed by a certificate authority (CA).
* The tool can automatically generate a new CA for you, or you can provide your own with the
-ca or -ca-cert command line options.
By default the 'cert' mode produces a single PKCS#12 output file which holds:
* The instance certificate
* The private key for the instance certificate
* The CA certificate
If you specify any of the following options:
* -pem (PEM formatted output)
* -keep-ca-key (retain generated CA key)
* -multiple (generate multiple certificates)
* -in (generate certificates from an input file)
then the output will be be a zip file containing individual certificate/key files
Enter password for CA (elastic-stack-ca.p12) :
Please enter the desired output file [elastic-certificates.p12]:
Enter password for elastic-certificates.p12 :
Certificates written to /usr/local/elk/elasticsearch-7.4.2/elastic-certificates.p12
This file should be properly secured as it contains the private key for
your instance.
This file is a self contained file and can be copied and used 'as is'
For each Elastic product that you wish to configure, you should copy
this '.p12' file to the relevant configuration directory
and then follow the SSL configuration instructions in the product guide.
For client applications, you may only need to copy the CA certificate and
configure the client to trust this certificate.
[elasticsearch@eus-image-design-es-kafka01:/usr/local/elk/elasticsearch-7.4.2]$ cp elastic-certificates.p12 config/
# 拷贝证书到其他节点，因为elasticsearch没有设置密码，用root用户传输，记得修改文件的权限
[root@eus-image-design-es-kafka01:/usr/local/elk/elasticsearch-7.4.2/config]# scp -P 2018 elastic-certificates.p12 172.30.0.107:/usr/local/elk/elasticsearch-7.4.2/config/
root@172.30.0.107's password:
elastic-certificates.p12 100% 3451 8.1MB/s 00:00
[root@eus-image-design-es-kafka01:/usr/local/elk/elasticsearch-7.4.2/config]# scp -P 2018 elastic-certificates.p12 172.30.0.110:/usr/local/elk/elasticsearch-7.4.2/config/
root@172.30.0.110's password:
elastic-certificates.p12
# chown elasticsearch.elasticsearch /usr/local/elk/elasticsearch-7.4.2/config/elastic-certificates.p12
# 修改配置，启用xpach认证，重新启动elasticserch集群
[elasticsearch@eus-image-design-es-kafka01:/usr/local/elk/elasticsearch-7.4.2]$ cat config/elasticsearch.yml
# 密码部分的配置最好等集群配置没有问题后再进行
xpack.security.enabled: true
xpack.security.transport.ssl.enabled: true
xpack.security.transport.ssl.verification_mode: certificate
xpack.security.transport.ssl.keystore.path: /usr/local/elk/elasticsearch-7.4.2/config/elastic-certificates.p12
xpack.security.transport.ssl.truststore.path: /usr/local/elk/elasticsearch-7.4.2/config/elastic-certificates.p12
# 此时不使用密码是没法查集群状态的
[root@eus-image-design-es-kafka01:/usr/local/elk/elasticsearch-7.4.2/config]# curl 172.30.0.106:9200/_cluster/health?pretty
{
"error" : {
"root_cause" : [
{
"type" : "security_exception",
"reason" : "missing authentication credentials for REST request [/_cluster/health?pretty]",
"header" : {
"WWW-Authenticate" : "Basic realm="security" charset="UTF-8""
}
}
],
"type" : "security_exception",
"reason" : "missing authentication credentials for REST request [/_cluster/health?pretty]",
"header" : {
"WWW-Authenticate" : "Basic realm="security" charset="UTF-8""
}
},
"status" : 401
}
# 设置集群密码
[elasticsearch@eus-image-design-es-kafka01:/usr/local/elk/elasticsearch-7.4.2]$ bin/elasticsearch-setup-passwords interactive
Initiating the setup of passwords for reserved users elastic,apm_system,kibana,logstash_system,beats_system,remote_monitoring_user.
You will be prompted to enter passwords as the process progresses.
Please confirm that you would like to continue [y/N]y
Enter password for [elastic]:
Reenter password for [elastic]:
Enter password for [apm_system]:
Reenter password for [apm_system]:
Enter password for [kibana]:
Reenter password for [kibana]:
Enter password for [logstash_system]:
Reenter password for [logstash_system]:
Enter password for [beats_system]:
Reenter password for [beats_system]:
Enter password for [remote_monitoring_user]:
Reenter password for [remote_monitoring_user]:
Changed password for user [apm_system]
Changed password for user [kibana]
Changed password for user [logstash_system]
Changed password for user [beats_system]
Changed password for user [remote_monitoring_user]
Changed password for user [elastic]
# 可以看到集群成功设置了密码，并且能够通过密码查看集群状态
[elasticsearch@eus-image-design-es-kafka01:/usr/local/elk/elasticsearch-7.4.2]$ curl -u elastic:pass 172.30.0.106:9200/_cluster/health?pretty
{
"cluster_name" : "online_image_es_pro",
"status" : "green",
"timed_out" : false,
"number_of_nodes" : 3,
"number_of_data_nodes" : 3,
"active_primary_shards" : 1,
"active_shards" : 2,
"relocating_shards" : 0,
"initializing_shards" : 0,
"unassigned_shards" : 0,
"delayed_unassigned_shards" : 0,
"number_of_pending_tasks" : 0,
"number_of_in_flight_fetch" : 0,
"task_max_waiting_in_queue_millis" : 0,
"active_shards_percent_as_number" : 100.0
}
# 添加监控项目
UserParameter=es_status,curl -u elastic:pass -sXGET http://172.30.0.106:9200/_cluster/health/?pretty | grep "status"|awk -F '[ "]+' '{print $4}'|grep -c 'green'
UserParameter=es_debug,sudo /bin/find /usr/local/elk/elasticsearch-7.4.2 -name hs_err_pid*.log -o -name java_pid*.hprof|wc -l
UserParameter=es_connect_nums,sudo netstat -tanolp|grep -v grep|grep ESTABLISHED|grep 9200|wc -l
# 触发脚本
[root@eus_influenex_es01:/usr/local/zabbix_agents/scripts]# more start_es.sh
#!/bin/bash
# if elasticsearch exists kill it
source /etc/profile
# 删除java报错产生的文件
/usr/bin/sudo /bin/find /usr/local/elk/elasticsearch-7.4.2 -name hs_err_pid*.log -o -name java_pid*.hprof | xargs rm -f
# kill并重新启动elasticsearch
count_es=`ps -ef|grep elasticsearch|grep -v grep|wc -l`
if [ $count_es -ge 1 ];then
ps -ef|grep elasticsearch|grep -v grep|/bin/kill -9 `awk '{print $2}'`
fi
# start it
su elasticsearch -c "cd /usr/local/elk/elasticsearch-7.4.2/bin && /bin/bash elasticsearch -d"
# 使用systemctl管理elasticsearch
vim /etc/systemd/system/elasticsearch.service
[Unit] Description=elasticsearch [Service] User=elasticsearch LimitNOFILE=500000 LimitNPROC=500000 LimitMEMLOCK=infinity ExecStart=/usr/local/elk/elasticsearch-7.4.2/bin/elasticsearch [Install] WantedBy=multi-user.target
报错：
memory locking requested for elasticsearch process but memory is not locked
参考配置：
https://www.elastic.co/guide/en/elasticsearch/reference/current/setting-system-settings.html#systemd