16.容器日志收集方案
- 把log-agent打包至业务镜像
- 日志落地至物理节点
- 每个物理节点启动日志容器
本次我们在每个node节点部署一个pod收集日志。
17.安装日志组件
设置serviceaccount
kubectl create serviceaccount admin -n kube-system
17.1 配置权限
mkdir /root/logs && cd /root/logs
[root@master01 logs]# cat es-rbac.yaml
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
name: es-rbac
subjects:
- kind: ServiceAccount
name: admin
namespace: kube-system
roleRef:
kind: ClusterRole
name: cluster-admin
apiGroup: rbac.authorization.k8s.io
[root@master01 logs]# kubectl apply -f es-rbac.yaml
clusterrolebinding.rbac.authorization.k8s.io/es-rbac created
17.2 安装elasticsearch
#在node节点pull镜像到本地
docker pull registry.cn-hangzhou.aliyuncs.com/cqz/elasticsearch:5.5.1
dpcker pull registry.cn-hangzhou.aliyuncs.com/acs-sample/kibana:5.5.1
docker pull registry.cn-hangzhou.aliyuncs.com/acs-sample/log-pilot:0.9-filebeat
修改vim elasticsearch.yml 主要是修改memory字段，测试环境适当改小内存使用，否则可能由于测试机的内存不够大服务起不来，生产环境可适当调大。
resources:
limits:
memory: 1500Mi
requests:
cpu: 100m
memory: 1000Mi
17.2.1 创建elasticsearch服务
[root@master01 logs]# kubectl apply -f elasticsearch.yml
service/elasticsearch-api created
service/elasticsearch-discovery created
statefulset.apps/elasticsearch created
[root@master01 logs]# kubectl get StatefulSet -n kube-system -o wide
NAME READY AGE CONTAINERS IMAGES
elasticsearch 2/2 9m37s elasticsearch registry.cn-hangzhou.aliyuncs.com/cqz/elasticsearch:5.5.1
17.2.2 查看ES状态
kubectl extc -it elasticsearch-0 bash -n kube-system
[root@master01 logs]# kubectl exec -it elasticsearch-0 bash -n kube-system
elasticsearch@elasticsearch-0:/usr/share/elasticsearch$ curl http://localhost:9200/_cat/health?v
epoch timestamp cluster status node.total node.data shards pri relo init unassign pending_tasks max_task_wait_time active_shards_percent
1597306845 08:20:45 docker-cluster green 2 2 0 0 0 0 0 0 - 100.0%
如果是出现error: unalbe to upgrade connection:Forbidden(user=system:anonymous,verb=create,resource=nodes,subresource=proxy)的错误
处理方法：
kubectl create clusterrolebinding system:anonymous --clusterrole=cluster-admin --user=system:anonymous
17.3 安装log-pilot 日志收集容器
kubectl apply -f log-pilot-2.0.yml
[root@master01 logs]# kubectl apply -f log-pilot-2.0.yml
daemonset.extensions/log-pilot created
17.4 安装kibana服务
[root@master01 logs]# kubectl apply -f kibana.yml
service/kibana created
deployment.apps/kibana created
17.5 获取kibana信息
[root@master01 logs]# kubectl get pod,svc -A -o wide | grep kibana
kube-system pod/kibana-777bb4dfb-js6gm 1/1 Running 0 34s 172.17.15.7 192.168.68.149 <none> <none>
kube-system service/kibana NodePort 10.0.0.225 <none> 80:36365/TCP 34s component=kibana
通过192.168.68.149:36365访问kibana页面
18 案例一：运行容器收集日志
18.1 创建nginx-demo.yaml文件
apiVersion: apps/v1
kind: Deployment
metadata:
creationTimestamp: null
labels:
app: nginx-demo
name: nginx-demo
spec:
replicas: 1
selector:
matchLabels:
app: nginx-demo
strategy: {}
template:
metadata:
creationTimestamp: null
labels:
app: nginx-demo
spec:
containers:
- image: nginx
name: nginx
resources: {}
env:
- name: aliyun_logs_nginx
value: "stdout"
status: {}
---
apiVersion: v1
kind: Service
metadata:
name: nginx-demo-svc
spec:
selector:
app: nginx-demo
ports:
- port: 80
targetPort: 80
说明：
其中aliyun_logs_nginx=stdout 表示要收集容器的stdout日志。--控制台输出 其中**aliyun_logs是固定字段，nginx为自定义变量。
aliyun_logs_access=/var/local/tomcat/logs/catalina..log表示要收集容器内/usr/local/tomcat/logs/目录下所有名字匹配catalina..log的文件日志。--日志文件输出
Log-pilot可以依据环境变量aliyun_logs_$name=$path 动态生成日志采集配置文件。
kubectl apply -f nginx-demo.yaml
[root@master01 nginx]# kubectl get pod,svc -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
pod/nginx-demo-7578b4d65-j22x9 1/1 Running 0 20s 172.17.15.8 192.168.68.149 <none> <none>
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE SELECTOR
service/nginx-demo-svc ClusterIP 10.0.0.136 <none> 80/TCP 20s app=nginx-demo
18.2 创建nginx-demo的ingress服务
cat >nginx-route.yaml<<EOF
apiVersion: traefik.containo.us/v1alpha1
kind: IngressRoute
metadata:
name: traefik-nginx-demo-route
spec:
entryPoints:
- web
routes:
- match: Host(\`nginx.cc.com\`)
kind: Rule
services:
- name: nginx-demo-ingress
port: 80
EOF
kubectl apply -f nginx-route.yaml
18.3 绑定host或使用serice访问
本地绑定hosts文件然后访问域名nginx.cc.com测试。
访问servce地址测试
[root@master01 nginx]# kubectl run -it --rm --restart=Never --image=infoblox/dnstools:latest dnstools
18.4 查看访问日志
[root@master01 nginx]# kubectl logs -f nginx-demo-7578b4d65-j22x9
/docker-entrypoint.sh: /docker-entrypoint.d/ is not empty, will attempt to perform configuration
/docker-entrypoint.sh: Looking for shell scripts in /docker-entrypoint.d/
/docker-entrypoint.sh: Launching /docker-entrypoint.d/10-listen-on-ipv6-by-default.sh
10-listen-on-ipv6-by-default.sh: Getting the checksum of /etc/nginx/conf.d/default.conf
10-listen-on-ipv6-by-default.sh: Enabled listen on IPv6 in /etc/nginx/conf.d/default.conf
/docker-entrypoint.sh: Launching /docker-entrypoint.d/20-envsubst-on-templates.sh
/docker-entrypoint.sh: Configuration complete; ready for start up
172.17.15.1 - - [14/Aug/2020:02:08:30 +0000] "GET / HTTP/1.1" 200 612 "-" "curl/7.60.0" "-"
172.17.15.2 - - [14/Aug/2020:02:11:01 +0000] "GET / HTTP/1.1" 200 612 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36" "192.168.25.208"
2020/08/14 02:11:01 [error] 29#29: *2 open() "/usr/share/nginx/html/favicon.ico" failed (2: No such file or directory), client: 172.17.15.2, server: localhost, request: "GET /favicon.ico HTTP/1.1", host: "nginx.cc.com", referrer: "http://nginx.cc.com/"
172.17.15.2 - - [14/Aug/2020:02:11:01 +0000] "GET /favicon.ico HTTP/1.1" 404 555 "http://nginx.cc.com/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36" "192.168.25.208"
172.17.15.2 - - [14/Aug/2020:02:25:01 +0000] "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36" "192.168.25.208"
172.17.15.2 - - [14/Aug/2020:02:25:06 +0000] "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36" "192.168.25.208"
18.5 查看是否建立索引
打开一个我们前面建立的elasticsearch的终端测试
kubectl exec -it elasticsearch-0 /bin/bash -n kube-system
curl 'localhost:9200/_cat/indices?v'
elasticsearch@elasticsearch-0:/usr/share/elasticsearch$ curl 'localhost:9200/_cat/indices?v'
health status index uuid pri rep docs.count docs.deleted store.size pri.store.size
green open .kibana REUfJToTR4-mHN-g8shDkA 1 1 1 0 6.4kb 3.2kb
green open nginx-2020.08.14 71bZtPrZSYWOyICuluQKNw 5 1 69 0 1mb 558.4kb
green open nginx-2020.08.13 VLL5cX4_Sduiv2b1rRnCew 5 1 7 0 110.7kb 55.3kb
18.6 将索引index写入到kibana中
通过kibana查看nginx访问日志