prometheus

https://github.com/kubernetes/kubernetes/tree/master/cluster/addons/prometheus
https://github.com/ikubernetes/k8s-prom

 1) 安装

git clone https://github.com/iKubernetes/k8s-prom.git
[root@master opt]# cd k8s-prom/
[root@master k8s-prom]# kubectl apply -f namespace.yaml
[root@master k8s-prom]# cd node_exporter/
[root@master node_exporter]# ls
node-exporter-ds.yaml  node-exporter-svc.yaml
[root@master node_exporter]# kubectl apply -f .
[root@master node_exporter]# kubectl get pods -n prom
NAME                             READY   STATUS    RESTARTS   AGE
prometheus-node-exporter-4h9jv   1/1     Running   0          76s
prometheus-node-exporter-hnwv9   1/1     Running   0          76s
prometheus-node-exporter-ngk2x   1/1     Running   0          76s
[root@master node_exporter]# cd ../prometheus/
[root@master prometheus]# kubectl apply -f .
configmap/prometheus-config created
deployment.apps/prometheus-server created
clusterrole.rbac.authorization.k8s.io/prometheus created
serviceaccount/prometheus created
clusterrolebinding.rbac.authorization.k8s.io/prometheus created
service/prometheus created
[root@master prometheus]# kubectl get all -n prom  查看该名称空间下的所有资源
[root@master prometheus]# kubectl get svc -n prom
NAME                       TYPE        CLUSTER-IP     EXTERNAL-IP   PORT(S)          AGE
prometheus                 NodePort    10.104.164.7   <none>        9090:30090/TCP   8m11s
prometheus-node-exporter   ClusterIP   None           <none>        9100/TCP         10m

 访问：http://192.168.10.12:30090/graph

[root@master k8s-prom]# cd kube-state-metrics/
[root@master kube-state-metrics]# kubectl apply -f .
deployment.apps/kube-state-metrics created
serviceaccount/kube-state-metrics created
clusterrole.rbac.authorization.k8s.io/kube-state-metrics created
clusterrolebinding.rbac.authorization.k8s.io/kube-state-metrics created
service/kube-state-metrics created
[root@master kube-state-metrics]# kubectl get pods -n prom
NAME                                  READY   STATUS             RESTARTS   AGE
kube-state-metrics-6b44579cc4-28cgf   0/1     ImagePullBackOff   0          3m9s
prometheus-node-exporter-4h9jv        1/1     Running            0          22m
prometheus-node-exporter-hnwv9        1/1     Running            0          22m
prometheus-node-exporter-ngk2x        1/1     Running            0          22m
prometheus-server-75cf46bdbc-z9kh7    1/1     Running            0          19m
[root@master kube-state-metrics]# kubectl describe pod kube-state-metrics-6b44579cc4-28cgf -n prom
  Warning  Failed     95s (x4 over 3m40s)   kubelet, node02    Error: ErrImagePull
  Normal   BackOff    71s (x6 over 3m40s)   kubelet, node02    Back-off pulling image "gcr.io/google_containers/kube-state-metrics-amd64:v1.3.1"
  Warning  Failed     58s (x7 over 3m40s)   kubelet, node02    Error: ImagePullBackOff
[root@master kube-state-metrics]# cat kube-state-metrics-deploy.yaml |grep image   # 更改镜像，重新应用
        image: quay.io/coreos/kube-state-metrics:v1.3.1
[root@master kube-state-metrics]# kubectl get pods -n prom
NAME                                  READY   STATUS    RESTARTS   AGE
kube-state-metrics-6697d66bbb-z9qsw   1/1     Running   0          48s
prometheus-node-exporter-4h9jv        1/1     Running   0          31m
prometheus-node-exporter-hnwv9        1/1     Running   0          31m
prometheus-node-exporter-ngk2x        1/1     Running   0          31m
prometheus-server-75cf46bdbc-z9kh7    1/1     Running   0          28m

创建证书

[root@master ~]# cd /etc/kubernetes/pki/
[root@master pki]# ls
apiserver.crt              apiserver.key                 ca.crt  front-proxy-ca.crt      front-proxy-client.key
apiserver-etcd-client.crt  apiserver-kubelet-client.crt  ca.key  front-proxy-ca.key      sa.key
apiserver-etcd-client.key  apiserver-kubelet-client.key  etcd    front-proxy-client.crt  sa.pub
[root@master pki]# (umask 077; openssl genrsa -out serving.key 2048)    #第一步
Generating RSA private key, 2048 bit long modulus
.+++
....+++
e is 65537 (0x10001)
[root@master pki]# openssl req -new -key serving.key -out serving.csr -subj "/CN=serving"        # 第二步
[root@master pki]# openssl x509 -req -in serving.csr -CA ./ca.crt -CAkey ./ca.key -CAcreateserial -out serving.crt -days 3650  # 第三步
Signature ok
subject=/CN=serving
Getting CA Private Key
[root@master pki]# ls serving.*
serving.crt  serving.csr  serving.key
[root@master k8s-prometheus-adapter]# cat custom-metrics-apiserver-deployment.yaml |grep secret
        secret:
          secretName: cm-adapter-serving-certs        # 需要用到的信息
[root@master pki]# kubectl create secret generic cm-adapter-serving-certs --from-file=serving.crt=./serving.crt --from-file=serving.key=./serving.key 
secret/cm-adapter-serving-certs created
[root@master pki]# kubectl get secret
NAME                       TYPE                                  DATA   AGE
cm-adapter-serving-certs   Opaque                                2      5m1s
default-token-q546g        kubernetes.io/service-account-token   3      44h
[root@master pki]# kubectl create secret generic cm-adapter-serving-certs --from-file=serving.crt=./serving.crt --from-file=serving.key=./serving.key -n prom
secret/cm-adapter-serving-certs created
[root@master pki]# kubectl get secret -n prom
NAME                             TYPE                                  DATA   AGE
cm-adapter-serving-certs         Opaque                                2      20s
default-token-wr4zt              kubernetes.io/service-account-token   3      161m
kube-state-metrics-token-9q4bt   kubernetes.io/service-account-token   3      140m
prometheus-token-kj5rj           kubernetes.io/service-account-token   3      157m

问题

[root@master k8s-prometheus-adapter]# pwd
/opt/k8s-prom/k8s-prometheus-adapter
[root@master k8s-prometheus-adapter]# kubectl apply -f .
[root@master k8s-prometheus-adapter]# kubectl get pods -n prom
NAME                                        READY   STATUS    RESTARTS   AGE
custom-metrics-apiserver-6bb45c6978-rl6zz   0/1     Error     2          54s
kube-state-metrics-6697d66bbb-z9qsw         1/1     Running   0          136m
prometheus-node-exporter-4h9jv              1/1     Running   0          167m
prometheus-node-exporter-hnwv9              1/1     Running   0          167m
prometheus-node-exporter-ngk2x              1/1     Running   0          167m
prometheus-server-75cf46bdbc-z9kh7          1/1     Running   0          164m

一波三折完成部署

[root@master k8s-prometheus-adapter]# pwd
/opt/k8s-prom/k8s-prometheus-adapter
[root@master k8s-prometheus-adapter]# kubectl apply -f .
[root@master k8s-prometheus-adapter]# kubectl get pods -n prom
NAME                                        READY   STATUS    RESTARTS   AGE
custom-metrics-apiserver-6bb45c6978-rl6zz   0/1     Error     2          54s
kube-state-metrics-6697d66bbb-z9qsw         1/1     Running   0          136m
prometheus-node-exporter-4h9jv              1/1     Running   0          167m
prometheus-node-exporter-hnwv9              1/1     Running   0          167m
prometheus-node-exporter-ngk2x              1/1     Running   0          167m
prometheus-server-75cf46bdbc-z9kh7          1/1     Running   0          164m
[root@master k8s-prometheus-adapter]# kubectl logs custom-metrics-apiserver-6bb45c6978-rl6zz -n prom 
unknown flag: --rate-interval   # 配置文件出现了未知的参数
    Usage of /adapter:
    unknown flag: --rate-interval
[root@master k8s-prometheus-adapter]# mv custom-metrics-apiserver-deployment.yaml{,.bak}
https://github.com/directxman12
[root@master k8s-prometheus-adapter]# wget https://raw.githubusercontent.com/DirectXMan12/k8s-prometheus-adapter/master/deploy/manifests/custom-metrics-apiserver-deployment.yaml
[root@master k8s-prometheus-adapter]# cp custom-metrics-apiserver-deployment.yaml{,.ori}  # 后面对比改了哪些内容
[root@master k8s-prometheus-adapter]# diff custom-metrics-apiserver-deployment.yaml custom-metrics-apiserver-deployment.yaml.ori
7c7
<   namespace: prom
---
>   namespace: custom-metrics
[root@master k8s-prometheus-adapter]# wget https://raw.githubusercontent.com/DirectXMan12/k8s-prometheus-adapter/master/deploy/manifests/custom-metrics-config-map.yaml
[root@master k8s-prometheus-adapter]# cp custom-metrics-config-map.yaml{,.ori}
[root@master k8s-prometheus-adapter]# diff custom-metrics-config-map.yaml{,.ori}
5c5
<   namespace: prom
---
>   namespace: custom-metrics
[root@master k8s-prometheus-adapter]# kubectl apply -f custom-metrics-config-map.yaml
configmap/adapter-config created
[root@master k8s-prometheus-adapter]# kubectl get cm -n prom
NAME                DATA   AGE
adapter-config      1      9s
prometheus-config   1      3h30m
[root@master k8s-prometheus-adapter]# kubectl apply -f custom-metrics-apiserver-deployment.yaml
deployment.apps/custom-metrics-apiserver created
[root@master k8s-prometheus-adapter]# kubectl get pods -n prom
NAME                                        READY   STATUS    RESTARTS   AGE
custom-metrics-apiserver-667fd4fffd-j5mzm   1/1     Running   0          12s
kube-state-metrics-6697d66bbb-z9qsw         1/1     Running   0          3h6m
prometheus-node-exporter-4h9jv              1/1     Running   0          3h37m
prometheus-node-exporter-hnwv9              1/1     Running   0          3h37m
prometheus-node-exporter-ngk2x              1/1     Running   0          3h37m
prometheus-server-75cf46bdbc-z9kh7          1/1     Running   0          3h34m
[root@master k8s-prometheus-adapter]# kubectl apply -f custom-metrics-apiserver-deployment.yaml
deployment.apps/custom-metrics-apiserver created
[root@master k8s-prometheus-adapter]# kubectl get pods -n prom
NAME                                        READY   STATUS    RESTARTS   AGE
custom-metrics-apiserver-667fd4fffd-j5mzm   1/1     Running   0          12s
kube-state-metrics-6697d66bbb-z9qsw         1/1     Running   0          3h6m
prometheus-node-exporter-4h9jv              1/1     Running   0          3h37m
prometheus-node-exporter-hnwv9              1/1     Running   0          3h37m
prometheus-node-exporter-ngk2x              1/1     Running   0          3h37m
prometheus-server-75cf46bdbc-z9kh7          1/1     Running   0          3h34m
[root@master k8s-prometheus-adapter]# 
[root@master k8s-prometheus-adapter]# kubectl get all -n prom

 一定要出现api

[root@master k8s-prometheus-adapter]# kubectl api-versions|grep custom
custom.metrics.k8s.io/v1beta1

第二种官方安装方式未测试

第二种方式
https://github.com/DirectXMan12/k8s-prometheus-adapter/tree/master/deploy/manifests
下载所以文件，应用。只修改名称空间，改为 prom

下一步将 普罗米修斯的资源导入grafana

安装服务

[root@master opt]# wget https://raw.githubusercontent.com/kubernetes-retired/heapster/master/deploy/kube-config/influxdb/grafana.yaml
[root@master opt]# cp grafana.yaml grafana.yaml.bak
[root@master opt]# diff grafana.yaml{,.bak}
5c5
<   namespace: prom
---
>   namespace: kube-system
16c16
<         image: registry.cn-hangzhou.aliyuncs.com/google_containers/heapster-grafana-amd64:v5.0.4
---
>         image: k8s.gcr.io/heapster-grafana-amd64:v5.0.4
27,28c27,28
<         #- name: INFLUXDB_HOST
<         #  value: monitoring-influxdb
---
>         - name: INFLUXDB_HOST
>           value: monitoring-influxdb
61c61
<   namespace: prom
---
>   namespace: kube-system
73d72
<   type: NodePort
[root@master opt]# kubectl apply -f grafana.yaml
deployment.extensions/monitoring-grafana created
service/monitoring-grafana created
[root@master opt]# kubectl get pods -n prom|grep monitoring-grafana
monitoring-grafana-75577776dc-47nzb         1/1     Running   0          2m22s

 查看svc的服务

[root@master opt]# kubectl get svc -n prom |grep monitoring-grafana
monitoring-grafana         NodePort    10.110.44.96    <none>        80:30351/TCP     4m52s

访问  http://192.168.10.12:30351

添加监控模板

https://grafana.com/dashboards
https://grafana.com/dashboards?search=kubernetes   # 搜索
https://grafana.com/dashboards/6417

下载json 文件存入到服务器
[root@master opt]# ls /opt/kubernetes-cluster-prometheus_rev1.json
/opt/kubernetes-cluster-prometheus_rev1.json

点击上面才会弹出下面导入的页面

最后将json 文件导入到grafana

 

导入下载的 json文件

1）服务器自动伸缩

[root@master opt]# kubectl explain hpa
KIND:     HorizontalPodAutoscaler
VERSION:  autoscaling/v1

[root@master opt]# kubectl explain hpa.spec

2）命令行运行资源限制

[root@master opt]# kubectl run myweb --image=ikubernetes/myapp:v1 --replicas=1 --requests='cpu=50m,memory=256Mi' --limits='cpu=50m,memory=256Mi' --labels='app=myweb' --expose --port=80
[root@master opt]# kubectl get pods
NAME                     READY   STATUS    RESTARTS   AGE
myapp-8w7bm              1/1     Running   2          3d2h
myapp-s9zmd              1/1     Running   2          136m
myweb-5dcbdcbf7f-5zhqt   1/1     Running   0          20s

 3）添加自动伸缩的服务

[root@master opt]# kubectl autoscale --help
[root@master opt]# kubectl autoscale deployment myweb --min=1 --max=8 --cpu-percent=60
horizontalpodautoscaler.autoscaling/myweb autoscaled
[root@master opt]# kubectl get hpa
NAME    REFERENCE          TARGETS         MINPODS   MAXPODS   REPLICAS   AGE
myweb   Deployment/myweb   <unknown>/60%   1         8         0          10s

4）为了让外界访问做压测

[root@master opt]# kubectl get svc
NAME         TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)        AGE
kubernetes   ClusterIP   10.96.0.1       <none>        443/TCP        3d4h
myapp        NodePort    10.99.99.99     <none>        80:30080/TCP   3d
myweb        ClusterIP   10.100.13.140   <none>        80/TCP         6m23s
[root@master opt]# kubectl patch svc myweb -p '{"spec":{"type":"NodePort"}}'
service/myweb patched
[root@master opt]# kubectl get svc
NAME         TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)        AGE
kubernetes   ClusterIP   10.96.0.1       <none>        443/TCP        3d4h
myapp        NodePort    10.99.99.99     <none>        80:30080/TCP   3d
myweb        NodePort    10.100.13.140   <none>        80:32106/TCP   7m11s

 5）ab 压测测试效果

安装ab命令
yum install httpd-tools -y
[root@master opt]# curl 192.168.10.12:32106/hostname.html
myweb-5dcbdcbf7f-5zhqt
[root@master opt]# ab -c 100 -n 500000 http://192.168.10.12:32106/hostname.html

[root@master ~]# kubectl describe hpa   查看资源描述
[root@master ~]# kubectl get pods
NAME                     READY   STATUS    RESTARTS   AGE
myapp-8w7bm              1/1     Running   2          3d2h
myapp-s9zmd              1/1     Running   2          152m
myweb-5dcbdcbf7f-5zhqt   1/1     Running   0          16m
myweb-5dcbdcbf7f-8k4nv   1/1     Running   0          37s
[root@master ~]# kubectl get pods
NAME                     READY   STATUS    RESTARTS   AGE
myapp-8w7bm              1/1     Running   2          3d2h
myapp-s9zmd              1/1     Running   2          152m
myweb-5dcbdcbf7f-5zhqt   1/1     Running   0          16m
myweb-5dcbdcbf7f-8k4nv   1/1     Running   0          84s
myweb-5dcbdcbf7f-jfwtz   1/1     Running   0          24s

6）hpa的yaml文件控制资源

[root@master hpa]# kubectl get hpa
NAME    REFERENCE          TARGETS   MINPODS   MAXPODS   REPLICAS   AGE
myweb   Deployment/myweb   0%/60%    1         8         1          14h
[root@master hpa]# kubectl delete hpa myweb
horizontalpodautoscaler.autoscaling "myweb" deleted
[root@master hpa]# kubectl get hpa
No resources found.
[root@master hpa]# cat  hpa-v2-demo.yaml 
apiVersion: autoscaling/v2beta1
kind: HorizontalPodAutoscaler
metadata:
  name: myweb-hpa-v2
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: myweb
  minReplicas: 1
  maxReplicas: 10
  metrics:
  - type: Resource
    resource:
      name: cpu
      targetAverageUtilization: 55
  - type: Resource
    resource:
      name: memory
      targetAverageValue: 50Mi
      
[root@master hpa]# kubectl apply -f hpa-v2-demo.yaml 
horizontalpodautoscaler.autoscaling/myweb-hpa-v2 created
[root@master ~]# kubectl get hpa
NAME           REFERENCE          TARGETS                MINPODS   MAXPODS   REPLICAS   AGE
myweb-hpa-v2   Deployment/myweb   7270400/50Mi, 0%/55%   1         10        1          93s
[root@master ~]# kubectl get hpa
NAME           REFERENCE          TARGETS                  MINPODS   MAXPODS   REPLICAS   AGE
myweb-hpa-v2   Deployment/myweb   7340032/50Mi, 102%/55%   1         10        2          3m45s
[root@master ~]# kubectl get pods
NAME                     READY   STATUS    RESTARTS   AGE
myapp-8w7bm              1/1     Running   2          3d17h
myapp-s9zmd              1/1     Running   2          17h
myweb-5dcbdcbf7f-5zhqt   1/1     Running   0          14h
myweb-5dcbdcbf7f-j65g7   1/1     Running   0          47s
myweb-5dcbdcbf7f-r972k   1/1     Running   0          108s