一、使用NFS配置持久化存储
1）在NFS服务器端（172.16.60.238）通过nfs创建es、filebeat共享目录
[root@k8s-harbor01 k8s]# mkdir -p /data/storage/k8s/es
2）创建NFS的rbac
[root@k8s-master01 k8s_project]# cd
[root@k8s-master01 ~]# cd /opt/k8s/k8s_project/
[root@k8s-master01 k8s_project]# mkdir elk
[root@k8s-master01 k8s_project]# cd elk/
[root@k8s-master01 elk]# vim nfs-rbac.yaml
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: nfs-provisioner
namespace: wiseco
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: nfs-provisioner-runner
namespace: wiseco
rules:
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "create", "delete"]
- apiGroups: [""]
resources: ["persistentvolumeclaims"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["storage.k8s.io"]
resources: ["storageclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["events"]
verbs: ["watch", "create", "update", "patch"]
- apiGroups: [""]
resources: ["services", "endpoints"]
verbs: ["get","create","list", "watch","update"]
- apiGroups: ["extensions"]
resources: ["podsecuritypolicies"]
resourceNames: ["nfs-provisioner"]
verbs: ["use"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: run-nfs-provisioner
subjects:
- kind: ServiceAccount
name: nfs-provisioner
namespace: wiseco
roleRef:
kind: ClusterRole
name: nfs-provisioner-runner
apiGroup: rbac.authorization.k8s.io
创建和查看
[root@k8s-master01 elk]# kubectl apply -f nfs-rbac.yaml serviceaccount/nfs-provisioner created clusterrole.rbac.authorization.k8s.io/nfs-provisioner-runner created clusterrolebinding.rbac.authorization.k8s.io/run-nfs-provisioner created [root@k8s-master01 elk]# kubectl get sa -n wiseco|grep nfs nfs-provisioner 1 4s [root@k8s-master01 elk]# kubectl get clusterrole -n wiseco|grep nfs nfs-provisioner-runner 2021-02-19T08:39:05Z [root@k8s-master01 elk]# kubectl get clusterrolebinding -n wiseco|grep nfs run-nfs-provisioner ClusterRole/nfs-provisioner-runner
二、ES集群部署
ES7.0+新版废弃了原先discovery.zen.ping.unicast.hosts及discovery.zen.minimum_master_nodes的探测方式，而是改为了discovery.seed_hosts及cluster.initial_master_nodes。
1）创建es集群的storage
[root@k8s-master01 elk]# pwd /opt/k8s/k8s_project/elk [root@k8s-master01 elk]# mkdir es [root@k8s-master01 elk]# cd es/ [root@k8s-master01 es]# vim es-nfs-class.yaml apiVersion: storage.k8s.io/v1beta1 kind: StorageClass metadata: name: es-nfs-storage namespace: wiseco provisioner: es/nfs reclaimPolicy: Retain
创建和查看
[root@k8s-master01 es]# kubectl apply -f es-nfs-class.yaml storageclass.storage.k8s.io/es-nfs-storage created [root@k8s-master01 es]# kubectl get sc -n wiseco NAME PROVISIONER RECLAIMPOLICY VOLUMEBINDINGMODE ALLOWVOLUMEEXPANSION AGE es-nfs-storage es/nfs Retain Immediate false 10s
2）创建es集群的nfs-client-provisioner
[root@k8s-master01 es]# vim es-nfs.yml
apiVersion: apps/v1
kind: Deployment
metadata:
name: es-nfs-client-provisioner
namespace: wiseco
spec:
replicas: 1
selector:
matchLabels:
app: es-nfs-client-provisioner
strategy:
type: Recreate
template:
metadata:
labels:
app: es-nfs-client-provisioner
spec:
serviceAccount: nfs-provisioner
containers:
- name: es-nfs-client-provisioner
image: registry.cn-hangzhou.aliyuncs.com/open-ali/nfs-client-provisioner
imagePullPolicy: IfNotPresent
volumeMounts:
- name: nfs-client-root
mountPath: /persistentvolumes
env:
- name: PROVISIONER_NAME
value: es/nfs
- name: NFS_SERVER
value: 172.16.60.238
- name: NFS_PATH
value: /data/storage/k8s/es
volumes:
- name: nfs-client-root
nfs:
server: 172.16.60.238
path: /data/storage/k8s/es
创建并查看
[root@k8s-master01 es]# kubectl apply -f es-nfs.yml deployment.apps/es-nfs-client-provisioner created [root@k8s-master01 es]# kubectl get pods -n wiseco|grep nfs es-nfs-client-provisioner-5c989d9b5-nkpdb 1/1 Running 0 4s
3）制作ES集群的镜像（jdk镜像、es镜像）
需要注意：
ES 7.6.2启动要求jdk要在java11以上版本，否则es启动会报错：
future versions of Elasticsearch will require Java 11; your Java version from [/usr/java/jdk1.8.0_192/jre] does not meet this requirement
接着制作es集群的镜像
下载elasticsearch-7.6.2-linux-x86_64.tar.gz安装包、准备elasticsearch.yml配置文件，这两个文件一起放到image镜像里。
[root@k8s-master01 images]# pwd
/opt/k8s/k8s_project/elk/es/images
[root@k8s-master01 images]# ll
total 0
drwxr-xr-x 2 root root 63 Feb 20 16:11 jdk_images
[root@k8s-master01 images]# mkdir es_images/
[root@k8s-master01 images]# ll
total 0
drwxr-xr-x 2 root root 96 Feb 20 15:49 es_images
drwxr-xr-x 2 root root 63 Feb 20 16:11 jdk_images
[root@k8s-master01 images]# cd es_images/
[root@k8s-master01 es_images]#
[root@k8s-master01 es_images]# wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.6.2-linux-x86_64.tar.gz
[root@k8s-master01 es_images]# ll
total 289540
-rw-r--r-- 1 root root 718 Feb 20 17:34 Dockerfile
-rw-r--r-- 1 root root 296477546 Mar 31 2020 elasticsearch-7.6.2-linux-x86_64.tar.gz
-rw-r--r-- 1 root root 448 Feb 20 17:49 elasticsearch.yml
这里千万要注意：node节点主机名要使用正确解析到的完整域名：pod名称.service名称.namespace名称.svc.cluster.local
[root@k8s-master01 es_images]# cat elasticsearch.yml
cluster.name: es-cluster
node.name: ${MY_POD_NAME}.es-svc.wiseco.svc.cluster.local
path.data: /opt/elasticsearch-7.6.2/data
path.logs: /opt/elasticsearch-7.6.2/logs
network.host: 0.0.0.0
http.port: 9200
http.cors.enabled: true
http.cors.allow-origin: "*"
node.master: true
node.data: true
discovery.seed_hosts: ["es-0.es-svc.wiseco.svc.cluster.local","es-1.es-svc.wiseco.svc.cluster.local","es-2.es-svc.wiseco.svc.cluster.local"]
cluster.initial_master_nodes: ["es-0.es-svc.wiseco.svc.cluster.local","es-1.es-svc.wiseco.svc.cluster.local","es-2.es-svc.wiseco.svc.cluster.local"]
镜像文件内容：
[root@k8s-master01 es_images]# cat Dockerfile
FROM 172.16.60.238/wiseco/jdk13.0.2
RUN rm -f /etc/localtime
&& ln -sv /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
&& echo "Asia/Shanghai" > /etc/timezone
ENV LANG en_US.UTF-8
ADD elasticsearch-7.6.2-linux-x86_64.tar.gz /opt
RUN mkdir -p /opt/elasticsearch-7.6.2/data
&& mkdir -p /opt/elasticsearch-7.6.2/logs
&& useradd elasticsearch
&& chown -R elasticsearch:elasticsearch /opt
&& chmod -R 777 /opt
&& setfacl -R -m u:elasticsearch:rwx /opt
&& setfacl -R -m u:elasticsearch:rwx /opt
&& rm -f /opt/elasticsearch-7.6.2/config/elasticsearch.yml
COPY elasticsearch.yml /opt/elasticsearch-7.6.2/config/
USER elasticsearch
EXPOSE 9200 9300
CMD ["/opt/elasticsearch-7.6.2/bin/elasticsearch"]
制作镜像并上传到Harbor仓库
[root@k8s-master01 images]# docker build -t 172.16.60.238/wiseco/elasticsearch-7.6.2 . [root@k8s-master01 images]# docker push 172.16.60.238/wiseco/elasticsearch-7.6.2
4）部署ES集群容器
注意：这里使用初始化容器来修改系统参数。
[root@k8s-master01 es]# pwd
/opt/k8s/k8s_project/elk/es
[root@k8s-master01 es]# mkdir deploy/
[root@k8s-master01 es]# cd deploy/
[root@k8s-master01 deploy]#
[root@k8s-master01 deploy]# cat es_cluster.yaml
apiVersion: v1
kind: Service
metadata:
name: es-svc
namespace: wiseco
labels:
app: es
spec:
ports:
- port: 9200
targetPort: 9200
name: outer
- port: 9300
targetPort: 9300
name: inner
clusterIP: None
selector:
app: es
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: es
namespace: wiseco
spec:
serviceName: "es-svc"
replicas: 3
selector:
matchLabels:
app: es
template:
metadata:
labels:
app: es
spec:
affinity:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchExpressions:
- key: "app"
operator: In
values:
- es
topologyKey: "kubernetes.io/hostname"
initContainers:
- name: increase-vm-max-map
image: busybox
command: ["sysctl", "-w", "vm.max_map_count=262144"]
securityContext:
privileged: true
- name: increase-fd-ulimit
image: busybox
command: ["sh", "-c", "ulimit -n 65536"]
securityContext:
privileged: true
terminationGracePeriodSeconds: 60
containers:
- name: es
image: 172.16.60.238/wiseco/elasticsearch-7.6.2
imagePullPolicy: Always
ports:
- containerPort: 9200
name: outer
- containerPort: 9300
name: inner
env:
- name: MY_POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
resources:
requests:
memory: 1024Mi
cpu: 500m
limits:
memory: 2048Mi
cpu: 1500m
lifecycle:
postStart:
exec:
command: ["/bin/sh","-c","touch /tmp/health"]
livenessProbe:
exec:
command: ["test","-e","/tmp/health"]
initialDelaySeconds: 5
timeoutSeconds: 5
periodSeconds: 10
readinessProbe:
tcpSocket:
port: outer
initialDelaySeconds: 15
timeoutSeconds: 5
periodSeconds: 20
volumeMounts:
- name: es-date
mountPath: /opt/elasticsearch-7.6.2/data
- name: es-log
mountPath: /opt/local/elasticsearch-7.6.2/logs
readOnly: false
volumes:
- name: es-log
hostPath:
path: /var/log/k8s-log/es
volumeClaimTemplates:
- metadata:
name: es-date
annotations:
volume.beta.kubernetes.io/storage-class: "es-nfs-storage"
spec:
accessModes:
- ReadWriteMany
resources:
requests:
storage: 10Gi
创建和查看
[root@k8s-master01 deploy]# kubectl apply -f es_cluster.yaml service/es-svc created statefulset.apps/es created [root@k8s-master01 deploy]# kubectl get pods -n wiseco -o wide NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES es-0 1/1 Running 0 9m36s 172.30.85.230 k8s-node01 <none> <none> es-1 1/1 Running 0 8m7s 172.30.217.85 k8s-node04 <none> <none> es-2 1/1 Running 0 6m1s 172.30.135.154 k8s-node03 <none> <none> [root@k8s-master01 deploy]# kubectl get svc -n wiseco|grep es es-svc ClusterIP None <none> 9200/TCP,9300/TCP 9m53s ingress-nginx NodePort 10.254.99.100 <none> 80:30080/TCP,443:30443/TCP 70d [root@k8s-master01 deploy]# kubectl get statefulset -n wiseco|grep es es 3/3 10m
5）查看NFS共享存储
NFS服务器（172.16.60.238），查看共享目录/data/storage/k8s/es
[root@k8s-harbor01 ~]# cd /data/storage/k8s/es/ [root@k8s-harbor01 es]# ll total 0 drwxrwxrwx 3 root root 19 Feb 21 21:36 wiseco-es-date-es-0-pvc-3607865a-ae40-4eeb-aa04-cf5ddab1599f drwxrwxrwx 3 root root 19 Feb 21 21:39 wiseco-es-date-es-1-pvc-3c1e1329-73b1-4477-b6b1-0b2c63e702d2 drwxrwxrwx 3 root root 19 Feb 21 21:40 wiseco-es-date-es-2-pvc-ecdff54e-2c28-4b33-8a5a-2e42c5b9c139 [root@k8s-harbor01 es]# ll ./* ./wiseco-es-date-es-0-pvc-3607865a-ae40-4eeb-aa04-cf5ddab1599f: total 0 drwxr-xr-x 3 1000 1000 15 Feb 21 21:36 nodes ./wiseco-es-date-es-1-pvc-3c1e1329-73b1-4477-b6b1-0b2c63e702d2: total 0 drwxr-xr-x 3 1000 1000 15 Feb 21 21:39 nodes ./wiseco-es-date-es-2-pvc-ecdff54e-2c28-4b33-8a5a-2e42c5b9c139: total 0 drwxr-xr-x 3 1000 1000 15 Feb 21 21:40 nodes
6）ES集群访问地址
ES集群在k8s内部访问地址：es-svc.wiseco.svc.cluster.local:9200
ES集群在k8s外部访问
需要配置ingress，提供一个外部访问的域名。比如：
[root@k8s-master01 ingress]# cat ingress.yml
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
name: wise-ingress
namespace: wiseco
annotations:
kubernetes.io/ingress.class: "nginx"
spec:
rules:
- host: elastic.kevin.com
http:
paths:
- backend:
serviceName: es-svc
servicePort: 9200
这样，在K8S集群外部访问此ES集群，访问地址为：http://elastic.kevin.com
7）ES集群连接和信息查看
可以登录到其中的一个es节点，进行es集群访问测试
[root@k8s-master01 deploy]# kubectl exec -ti es-0 -n wiseco -- /bin/bash
[elasticsearch@es-0 /]$ curl http://es-svc.wiseco.svc.cluster.local:9200
{
"name" : "es-0.es-svc.wiseco.svc.cluster.local",
"cluster_name" : "es-cluster",
"cluster_uuid" : "K-AFavs-RaKjq60rMQG1WQ",
"version" : {
"number" : "7.6.2",
"build_flavor" : "default",
"build_type" : "tar",
"build_hash" : "ef48eb35cf30adf4db14086e8aabd07ef6fb113f",
"build_date" : "2020-03-26T06:34:37.794943Z",
"build_snapshot" : false,
"lucene_version" : "8.4.0",
"minimum_wire_compatibility_version" : "6.8.0",
"minimum_index_compatibility_version" : "6.0.0-beta1"
},
"tagline" : "You Know, for Search"
}
查看ES集群状态
[elasticsearch@es-0 /]$ curl -XGET "http://es-svc.wiseco.svc.cluster.local:9200/_cat/nodes"
172.30.135.154 11 54 2 0.13 0.31 0.25 dilm - es-2.es-svc.wiseco.svc.cluster.local
172.30.85.230 10 66 2 0.11 0.29 0.46 dilm * es-0.es-svc.wiseco.svc.cluster.local
172.30.217.85 6 65 1 0.22 0.21 0.21 dilm - es-1.es-svc.wiseco.svc.cluster.local
查看集群详细信息，后面添加"?v"
注意：带*符号的表示是当前的master主节点
[elasticsearch@es-0 /]$ curl -XGET 'http://es-svc.wiseco.svc.cluster.local:9200/_cat/nodes?v'
ip heap.percent ram.percent cpu load_1m load_5m load_15m node.role master name
172.30.135.154 12 54 2 0.09 0.29 0.25 dilm - es-2.es-svc.wiseco.svc.cluster.local
172.30.85.230 12 66 1 0.07 0.27 0.45 dilm * es-0.es-svc.wiseco.svc.cluster.local
172.30.217.85 8 65 3 0.22 0.21 0.21 dilm - es-1.es-svc.wiseco.svc.cluster.local
查询集群状态方法
[elasticsearch@es-0 /]$ curl -XGET 'http://es-svc.wiseco.svc.cluster.local:9200/_cluster/state/nodes?pretty'
{
"cluster_name" : "es-cluster",
"cluster_uuid" : "K-AFavs-RaKjq60rMQG1WQ",
"nodes" : {
"HTD4h0xZRcO3uypCzmxfpQ" : {
"name" : "es-2.es-svc.wiseco.svc.cluster.local",
"ephemeral_id" : "OyTHKFt9Ska6-XGp1ucRsQ",
"transport_address" : "172.30.135.154:9300",
"attributes" : {
"ml.machine_memory" : "8370167808",
"ml.max_open_jobs" : "20",
"xpack.installed" : "true"
}
},
"ZyL8_6pHTjOwLXCzJDRyVw" : {
"name" : "es-0.es-svc.wiseco.svc.cluster.local",
"ephemeral_id" : "JATRby-lTzicDZb9oBoqUQ",
"transport_address" : "172.30.85.230:9300",
"attributes" : {
"ml.machine_memory" : "8366628864",
"xpack.installed" : "true",
"ml.max_open_jobs" : "20"
}
},
"KGQQtVrqR0CeajjARZY4LQ" : {
"name" : "es-1.es-svc.wiseco.svc.cluster.local",
"ephemeral_id" : "5NKITiKTTMWUI0QAXXX6qg",
"transport_address" : "172.30.217.85:9300",
"attributes" : {
"ml.machine_memory" : "8370184192",
"ml.max_open_jobs" : "20",
"xpack.installed" : "true"
}
}
}
}
查询集群中的master（下面两个命令都可以）
[elasticsearch@es-0 /]$ curl -XGET 'http://es-svc.wiseco.svc.cluster.local:9200/_cluster/state/master_node?pretty'
{
"cluster_name" : "es-cluster",
"cluster_uuid" : "K-AFavs-RaKjq60rMQG1WQ",
"master_node" : "ZyL8_6pHTjOwLXCzJDRyVw"
}
[elasticsearch@es-0 /]$ curl -XGET 'http://es-svc.wiseco.svc.cluster.local:9200/_cat/master?v'
id host ip node
ZyL8_6pHTjOwLXCzJDRyVw 172.30.85.230 172.30.85.230 es-0.es-svc.wiseco.svc.cluster.local
查询集群的健康状态（一共三种状态：green、yellow，red；其中green表示健康）
下面两个命令都可以
[elasticsearch@es-0 /]$ curl -XGET 'http://es-svc.wiseco.svc.cluster.local:9200/_cat/health?v'
epoch timestamp cluster status node.total node.data shards pri relo init unassign pending_tasks max_task_wait_time active_shards_percent
1613915456 13:50:56 es-cluster green 3 3 0 0 0 0 0 0 - 100.0%
[elasticsearch@es-0 /]$ curl -XGET 'http://es-svc.wiseco.svc.cluster.local:9200/_cluster/health?pretty'
{
"cluster_name" : "es-cluster",
"status" : "green",
"timed_out" : false,
"number_of_nodes" : 3,
"number_of_data_nodes" : 3,
"active_primary_shards" : 0,
"active_shards" : 0,
"relocating_shards" : 0,
"initializing_shards" : 0,
"unassigned_shards" : 0,
"delayed_unassigned_shards" : 0,
"number_of_pending_tasks" : 0,
"number_of_in_flight_fetch" : 0,
"task_max_waiting_in_queue_millis" : 0,
"active_shards_percent_as_number" : 100.0
}