EFK部署
一、安装elasticsearch集群
1）创建名称空间
[root@k8s-master1 efk]# kubectl create namespace kube-logging
namespace/kube-logging created
[root@k8s-master1 efk]# kubectl get ns
NAME STATUS AGE
default Active 4d17h
kube-logging Active 3s
kube-node-lease Active 4d17h
kube-public Active 4d17h
kube-system Active 4d17h
kubernetes-dashboard Active 4d16h
2）创建headless service服务
[root@k8s-master1 efk]# vim elasticsearch_svc.yaml
kind: Service
apiVersion: v1
metadata:
name: elasticsearch
namespace: kube-logging
labels:
app: elasticsearch
spec:
selector:
app: elasticsearch
clusterIP: None
ports:
- port: 9200
name: rest
- port: 9300
name: inter-node
# 更新
[root@k8s-master1 efk]# kubectl apply -f elasticsearch_svc.yaml
[root@k8s-master1 efk]# kubectl get svc -n kube-logging
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
elasticsearch ClusterIP None <none> 9200/TCP,9300/TCP 9s
[root@k8s-master1 efk]# kubectl describe svc -n kube-logging elasticsearch
Name: elasticsearch
Namespace: kube-logging
Labels: app=elasticsearch
Annotations: <none>
Selector: app=elasticsearch
Type: ClusterIP
IP Families: <none>
IP: None
IPs: None
Port: rest 9200/TCP
TargetPort: 9200/TCP
Endpoints: <none>
Port: inter-node 9300/TCP
TargetPort: 9300/TCP
Endpoints: <none>
Session Affinity: None
Events: <none>
3）创建Storageclass，实现存储类动态供给
安装nfs：
# yum安装nfs
[root@k8s-master1 ~]# yum install nfs-utils -y
[root@k8s-node1 ~]# yum install nfs-utils -y
[root@k8s-node2 ~]# yum install nfs-utils -y
# 启动nfs服务
[root@k8s-master1 ~]# systemctl start nfs && systemctl enable nfs.service
[root@k8s-node1 ~]# systemctl start nfs && systemctl enable nfs.service
[root@k8s-node2 ~]# systemctl start nfs && systemctl enable nfs.service
# 在k8s-master1上创建一个nfs共享目录
[root@k8s-master1 ~]# mkdir /data/v1 -p
# 编辑/etc/exports文件
[root@k8s-master1 ~]# vim /etc/exports
/data/v1 192.168.40.0/24(rw,no_root_squash)
# 加载配置，使配置生效
[root@k8s-master1 ~]# exportfs -arv
[root@k8s-master1 ~]# systemctl restart nfs
创建nfs作为存储的供应商：
# 1、创建sa
[root@k8s-master1 efk]# cat serviceaccount.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
name: nfs-provisioner
[root@k8s-master1 efk]# kubectl apply -f serviceaccount.yaml
serviceaccount/nfs-provisioner created
[root@k8s-master1 efk]# kubectl get sa
NAME SECRETS AGE
default 1 4d17h
nfs-provisioner 1 6s
# 2、对sa做rbac授权
[root@k8s-master1 efk]# cat rbac.yaml
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: nfs-provisioner-runner
rules:
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "create", "delete"]
- apiGroups: [""]
resources: ["persistentvolumeclaims"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["storage.k8s.io"]
resources: ["storageclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["events"]
verbs: ["create", "update", "patch"]
- apiGroups: [""]
resources: ["services", "endpoints"]
verbs: ["get"]
- apiGroups: ["extensions"]
resources: ["podsecuritypolicies"]
resourceNames: ["nfs-provisioner"]
verbs: ["use"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: run-nfs-provisioner
subjects:
- kind: ServiceAccount
name: nfs-provisioner
namespace: default
roleRef:
kind: ClusterRole
name: nfs-provisioner-runner
apiGroup: rbac.authorization.k8s.io
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: leader-locking-nfs-provisioner
rules:
- apiGroups: [""]
resources: ["endpoints"]
verbs: ["get", "list", "watch", "create", "update", "patch"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: leader-locking-nfs-provisioner
subjects:
- kind: ServiceAccount
name: nfs-provisioner
namespace: default
roleRef:
kind: Role
name: leader-locking-nfs-provisioner
apiGroup: rbac.authorization.k8s.io
[root@k8s-master1 efk]# kubectl apply -f rbac.yaml
注意：k8s1.20+版本通过nfs provisioner动态生成pv会报错：Unexpected error getting claim reference to claim "default/test-claim1": selfLink was empty, can't make reference，报错原因是1.20版本启用了selfLink，解决方法如下；
# 编辑/etc/kubernetes/manifests/kube-apiserver.yaml,添加这一行：
# - --feature-gates=RemoveSelfLink=false
[root@k8s-master1 efk]# vim /etc/kubernetes/manifests/kube-apiserver.yaml
spec:
containers:
- command:
- kube-apiserver
- --feature-gates=RemoveSelfLink=false # 添加这一行
- --advertise-address=192.168.40.180
# 更新kube-apiserver.yaml文件
[root@k8s-master1 efk]# kubectl apply -f /etc/kubernetes/manifests/kube-apiserver.yaml
pod/kube-apiserver created
# 删除CrashLoopBackOff的pod
[root@k8s-master1 efk]# kubectl get pods -n kube-system | grep apiserver
kube-apiserver 0/1 CrashLoopBackOff 1 19s
kube-apiserver-k8s-master1 1/1 Running 0 45s
[root@k8s-master1 efk]# kubectl delete pods kube-apiserver -n kube-system
pod "kube-apiserver" deleted
通过deployment创建pod用来运行nfs-provisioner：
[root@k8s-master1 efk]# cat deployment.yaml
kind: Deployment
apiVersion: apps/v1
metadata:
name: nfs-provisioner
spec:
selector:
matchLabels:
app: nfs-provisioner
replicas: 1
strategy:
type: Recreate
template:
metadata:
labels:
app: nfs-provisioner
spec:
serviceAccount: nfs-provisioner
containers:
- name: nfs-provisioner
image: registry.cn-hangzhou.aliyuncs.com/open-ali/xianchao/nfs-client-provisioner:v1
imagePullPolicy: IfNotPresent
volumeMounts:
- name: nfs-client-root
mountPath: /persistentvolumes
env:
- name: PROVISIONER_NAME
value: example.com/nfs
- name: NFS_SERVER
value: 192.168.40.180
- name: NFS_PATH
value: /data/v1
volumes:
- name: nfs-client-root
nfs:
server: 192.168.40.180
path: /data/v1
[root@k8s-master1 efk]# kubectl apply -f deployment.yaml
deployment.apps/nfs-provisioner created
[root@k8s-master1 ~]# kubectl get pods | grep nfs
nfs-provisioner-577487c6f6-bbglv 1/1 Running 1 43s
创建stoorageclass：
[root@k8s-master1 efk]# cat class.yaml
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: do-block-storage
provisioner: example.com/nfs # 该值需要和nfs provisioner配置的PROVISIONER_NAME处的value值保持一致
[root@k8s-master1 efk]# kubectl apply -f class.yaml
storageclass.storage.k8s.io/do-block-storage created
[root@k8s-master1 efk]# kubectl get storageclasses
NAME PROVISIONER RECLAIMPOLICY VOLUMEBINDINGMODE ALLOWVOLUMEEXPANSION AGE
do-block-storage example.com/nfs Delete Immediate false 32s
4）安装elasticsearch集群
[root@k8s-master1 efk]# cat elasticsearch-statefulset.yaml
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: es-cluster
namespace: kube-logging
spec:
serviceName: elasticsearch
replicas: 3
selector:
matchLabels:
app: elasticsearch
template:
metadata:
labels:
app: elasticsearch
spec:
containers:
- name: elasticsearch
image: docker.elastic.co/elasticsearch/elasticsearch:7.2.0
imagePullPolicy: IfNotPresent
resources:
limits:
cpu: 1000m
requests:
cpu: 100m
ports:
- containerPort: 9200
name: rest
protocol: TCP
- containerPort: 9300
name: inter-node
protocol: TCP
volumeMounts:
- name: data
mountPath: /usr/share/elasticsearch/data
env:
- name: cluster.name
value: k8s-logs
- name: node.name
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: discovery.seed_hosts
value: "es-cluster-0.elasticsearch.kube-logging.svc.cluster.local,es-cluster-1.elasticsearch.kube-logging.svc.cluster.local,es-cluster-2.elasticsearch.kube-logging.svc.cluster.local"
- name: cluster.initial_master_nodes
value: "es-cluster-0,es-cluster-1,es-cluster-2"
- name: ES_JAVA_OPTS
value: "-Xms512m -Xmx512m"
initContainers:
- name: fix-permissions
image: busybox
imagePullPolicy: IfNotPresent
command: ["sh", "-c", "chown -R 1000:1000 /usr/share/elasticsearch/data"]
securityContext:
privileged: true
volumeMounts:
- name: data
mountPath: /usr/share/elasticsearch/data
- name: increase-vm-max-map
image: busybox
imagePullPolicy: IfNotPresent
command: ["sysctl", "-w", "vm.max_map_count=262144"]
securityContext:
privileged: true
- name: increase-fd-ulimit
image: busybox
imagePullPolicy: IfNotPresent
command: ["sh", "-c", "ulimit -n 65536"]
securityContext:
privileged: true
volumeClaimTemplates:
- metadata:
name: data
labels:
app: elasticsearch
spec:
accessModes: [ "ReadWriteOnce" ]
storageClassName: do-block-storage
resources:
requests:
storage: 10Gi
[root@k8s-master1 efk]# kubectl apply -f elasticsearch-statefulset.yaml
[root@k8s-master1 efk]# kubectl get pods -n kube-logging -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
es-cluster-0 1/1 Running 0 15s 10.244.169.141 k8s-node2 <none> <none>
es-cluster-1 1/1 Running 0 10s 10.244.169.142 k8s-node2 <none> <none>
es-cluster-2 1/1 Running 0 5s 10.244.169.143 k8s-node2 <none> <none>
[root@k8s-master1 efk]# kubectl get svc -n kube-logging
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
elasticsearch ClusterIP None <none> 9200/TCP,9300/TCP 30m
[root@k8s-master1 efk]# kubectl describe svc -n kube-logging elasticsearch
Name: elasticsearch
Namespace: kube-logging
Labels: app=elasticsearch
Annotations: <none>
Selector: app=elasticsearch
Type: ClusterIP
IP Families: <none>
IP: None
IPs: None
Port: rest 9200/TCP
TargetPort: 9200/TCP
Endpoints: 10.244.169.141:9200,10.244.169.142:9200,10.244.169.143:9200
Port: inter-node 9300/TCP
TargetPort: 9300/TCP
Endpoints: 10.244.169.141:9300,10.244.169.142:9300,10.244.169.143:9300
Session Affinity: None
Events: <none>
5）检查elasticsearch集群是否部署成功
# 使用下面的命令将本地端口9200转发到 Elasticsearch 节点（如es-cluster-0）对应的端口
[root@k8s-master1 efk]# kubectl port-forward es-cluster-0 9200:9200 --namespace=kube-logging
Forwarding from 127.0.0.1:9200 -> 9200
Forwarding from [::1]:9200 -> 9200
Handling connection for 9200
...
# 另外的终端窗口中，执行如下请求，新开一个master1终端
[root@k8s-master1 ~]# curl http://localhost:9200/_cluster/state?pretty
{
"cluster_name" : "k8s-logs",
"cluster_uuid" : "HnPjL3TQQgKZz_mh5f37_w",
"version" : 21,
"state_uuid" : "n1Fb-o37S1KOvkfo6uAQMA",
"master_node" : "lf5Cs37LRjOvNgz0qc6uRQ",
"blocks" : { },
"nodes" : {
"ZaP5NWM7RfGgUL_SG05lxg" : {
"name" : "es-cluster-1",
"ephemeral_id" : "P_CbcTgpQ22pIcD34cMsTA",
"transport_address" : "10.244.169.142:9300",
"attributes" : {
"ml.machine_memory" : "4126896128",
"ml.max_open_jobs" : "20",
"xpack.installed" : "true"
}
},
"lf5Cs37LRjOvNgz0qc6uRQ" : {
"name" : "es-cluster-0",
"ephemeral_id" : "mC251P-_Ssain17gquAvvw",
"transport_address" : "10.244.169.141:9300",
"attributes" : {
"ml.machine_memory" : "4126896128",
"xpack.installed" : "true",
"ml.max_open_jobs" : "20"
}
},
"BEPuaYxpRqmP9g1lJFtKEQ" : {
"name" : "es-cluster-2",
"ephemeral_id" : "O0_3J6miTRmevIqmIAO3Zg",
"transport_address" : "10.244.169.143:9300",
"attributes" : {
"ml.machine_memory" : "4126896128",
"ml.max_open_jobs" : "20",
"xpack.installed" : "true"
}
}
},
.....
}
二、安装Kibanan可视化UI界面
1）创建kibanan资源清单
[root@k8s-master1 efk]# cat kibana.yaml
apiVersion: v1
kind: Service
metadata:
name: kibana
namespace: kube-logging
labels:
app: kibana
spec:
ports:
- port: 5601
selector:
app: kibana
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: kibana
namespace: kube-logging
labels:
app: kibana
spec:
replicas: 1
selector:
matchLabels:
app: kibana
template:
metadata:
labels:
app: kibana
spec:
containers:
- name: kibana
image: docker.elastic.co/kibana/kibana:7.2.0
imagePullPolicy: IfNotPresent
resources:
limits:
cpu: 1000m
requests:
cpu: 100m
env:
- name: ELASTICSEARCH_URL
value: http://elasticsearch.kube-logging.svc.cluster.local:9200
ports:
- containerPort: 5601
[root@k8s-master1 efk]# kubectl apply -f kibana.yaml
[root@k8s-master1 efk]# kubectl get svc -n kube-logging
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
elasticsearch ClusterIP None <none> 9200/TCP,9300/TCP 63m
kibana ClusterIP 10.107.91.208 <none> 5601/TCP 23s
[root@k8s-master1 efk]# kubectl get pods -n kube-logging
NAME READY STATUS RESTARTS AGE
es-cluster-0 1/1 Running 0 31m
es-cluster-1 1/1 Running 0 31m
es-cluster-2 1/1 Running 0 31m
kibana-66f59798b7-w4gd4 1/1 Running 0 33s
# 修改service的type类型为NodePort
[root@k8s-master1 efk]# kubectl edit svc kibana -n kube-logging
type: NodePort
[root@k8s-master1 efk]# kubectl get svc -n kube-logging
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
elasticsearch ClusterIP None <none> 9200/TCP,9300/TCP 65m
kibana NodePort 10.107.91.208 <none> 5601:31089/TCP 2m3s
在浏览器中打开http://<任意节点IP>:31089即可，如果看到如下欢迎界面证明 Kibana 已经成功部署到了Kubernetes集群之中。
三、部署Fluentd
使用daemonset控制器部署fluentd组件，这样可以保证集群中的每个节点都可以运行同样fluentd的pod副本，这样就可以收集k8s集群中每个节点的日志，在k8s集群中，容器应用程序的输入输出日志会重定向到node节点里的json文件中，fluentd可以tail和过滤以及把日志转换成指定的格式发送到elasticsearch集群中。除了容器日志，fluentd也可以采集kubelet、kube-proxy、docker的日志。
1）准备资源清单
[root@k8s-master1 efk]# cat fluentd.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
name: fluentd
namespace: kube-logging
labels:
app: fluentd
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: fluentd
labels:
app: fluentd
rules:
- apiGroups:
- ""
resources:
- pods
- namespaces
verbs:
- get
- list
- watch
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: fluentd
roleRef:
kind: ClusterRole
name: fluentd
apiGroup: rbac.authorization.k8s.io
subjects:
- kind: ServiceAccount
name: fluentd
namespace: kube-logging
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: fluentd
namespace: kube-logging
labels:
app: fluentd
spec:
selector:
matchLabels:
app: fluentd
template:
metadata:
labels:
app: fluentd
spec:
serviceAccount: fluentd
serviceAccountName: fluentd
tolerations:
- key: node-role.kubernetes.io/master
effect: NoSchedule
containers:
- name: fluentd
image: fluent/fluentd-kubernetes-daemonset:v1.4.2-debian-elasticsearch-1.1
imagePullPolicy: IfNotPresent
env:
- name: FLUENT_ELASTICSEARCH_HOST
value: "elasticsearch.kube-logging.svc.cluster.local"
- name: FLUENT_ELASTICSEARCH_PORT
value: "9200"
- name: FLUENT_ELASTICSEARCH_SCHEME
value: "http"
- name: FLUENTD_SYSTEMD_CONF
value: disable
resources:
limits:
memory: 512Mi
requests:
cpu: 100m
memory: 200Mi
volumeMounts:
- name: varlog
mountPath: /var/log
- name: varlibdockercontainers
mountPath: /var/lib/docker/containers
readOnly: true
terminationGracePeriodSeconds: 30
volumes:
- name: varlog
hostPath:
path: /var/log
- name: varlibdockercontainers
hostPath:
path: /var/lib/docker/containers
# 更新
[root@k8s-master1 efk]# kubectl apply -f fluentd.yaml
[root@k8s-master1 efk]# kubectl get pods -n kube-logging
NAME READY STATUS RESTARTS AGE
es-cluster-0 1/1 Running 0 38m
es-cluster-1 1/1 Running 0 38m
es-cluster-2 1/1 Running 0 38m
fluentd-nkvqj 1/1 Running 0 33s
fluentd-vpsgk 1/1 Running 0 33s
fluentd-vrwjw 1/1 Running 0 33s
kibana-66f59798b7-w4gd4 1/1 Running 0 7m43s
2）配置kibanna
点击左侧的Discover，在这里可以配置我们需要的 Elasticsearch 索引，前面 Fluentd 配置文件中我们采集的日志使用的是 logstash 格式，这里只需要在文本框中输入logstash-*即可匹配到 Elasticsearch 集群中的所有日志数据
点击Next step，选择@timestamp，创建索引
点击左侧的discover，可看到如下：
四、测试收集容器日志
1）创建容器
[root@k8s-master1 efk]# cat pod.yaml
apiVersion: v1
kind: Pod
metadata:
name: counter
spec:
containers:
- name: count
image: busybox
imagePullPolicy: IfNotPresent
args: [/bin/sh, -c,'i=0; while true; do echo "$i: $(date)"; i=$((i+1)); sleep 1; done']
[root@k8s-master1 efk]# kubectl apply -f pod.yaml
pod/counter created
[root@k8s-master1 efk]# kubectl get pods
NAME READY STATUS RESTARTS AGE
counter 1/1 Running 0 9s
nfs-provisioner-577487c6f6-bbglv 1/1 Running 1 57m
2）登录到kibana的控制面板，在discover处的搜索栏中输入kubernetes.pod_name:counter，这将过滤名为的Pod的日志数据counter
Kibana查询语言KQL官方地址：https://www.elastic.co/guide/en/kibana/7.2/kuery-query.html
