helm安装prometheus-operator

一、环境介绍

[root@master test]# uname -r
4.4.223-1.el7.elrepo.x86_64
[root@master test]# kubectl  get node 
NAME     STATUS   ROLES    AGE   VERSION
master   Ready    master   27d   v1.18.0
node     Ready    <none>   27d   v1.18.0
[root@master test]# helm  version
version.BuildInfo{Version:"v3.2.0", GitCommit:"e11b7ce3b12db2941e90399e874513fbd24bcb71", GitTreeState:"clean", GoVersion:"go1.13.10"}

　　

二、helm添加仓库

#阿里云
helm  repo add aliyuncs https://apphub.aliyuncs.com
#官方
helm  repo add stable  https://kubernetes-charts.storage.googleapis.com

三、helm search prometheus-operator

[root@master test]# helm search repo  prometheus-operator
NAME                            CHART VERSION    APP VERSION    DESCRIPTION                                       
aliyuncs/prometheus-operator    8.7.0            0.35.0         Provides easy monitoring definitions for Kubern...
stable/prometheus-operator      8.13.7           0.38.1         Provides easy monitoring definitions for Kubern...

四、安装

helm  install mypro aliyuncs/prometheus-operator 

五、查看

[root@master test]# helm list
NAME     NAMESPACE    REVISION    UPDATED                                    STATUS      CHART                        APP VERSION
mypro    default      1           2020-06-09 09:32:37.091220013 +0800 CST    deployed    prometheus-operator-8.7.0    0.35.0     
[root@master test]# helm  status mypro
NAME: mypro
LAST DEPLOYED: Tue Jun  9 09:32:37 2020
NAMESPACE: default
STATUS: deployed
REVISION: 1
NOTES:
The Prometheus Operator has been installed. Check its status by running:
  kubectl --namespace default get pods -l "release=mypro"

Visit https://github.com/coreos/prometheus-operator for instructions on how
to create & configure Alertmanager and Prometheus instances using the Operator.
[root@master test]# kubectl --namespace default get pods -l "release=mypro"
NAME                                                  READY   STATUS    RESTARTS   AGE
mypro-grafana-f5b868868-8ckgs                         2/2     Running   0          55m
mypro-prometheus-node-exporter-dg6w4                  1/1     Running   0          55m
mypro-prometheus-node-exporter-x9l4b                  1/1     Running   0          55m
mypro-prometheus-operator-operator-5b458d4659-p7t4l   2/2     Running   0          55m

六、配置ingress浏览器访问

[root@master test]# cat grafana-ingress.yaml 
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  name: ingress-grafana
spec:
  rules:
  - host: grafana.zhang.com
    http:
      paths:
      - backend:
          serviceName: mypro-grafana
          servicePort: 80
[root@master test]# cat prometheus-ingress.yaml 
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  name: ingress-prometheus
spec:
  rules:
  - host: prometheus.zhang.com
    http:
      paths:
      - backend:
          serviceName: mypro-prometheus-operator-prometheus
          servicePort: 9090
[root@master test]# cat alertmanager-ingress.yaml 
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  name: ingress-alertmanager
spec:
  rules:
  - host: alertmanager.zhang.com
    http:
      paths:
      - backend:
          serviceName: mypro-prometheus-operator-alertmanager
          servicePort: 9093

七、查看

[root@master test]# kubectl get ingress
NAME                   CLASS    HOSTS                    ADDRESS         PORTS   AGE
ingress-alertmanager   <none>   alertmanager.zhang.com   10.111.12.239   80      29m
ingress-grafana        <none>   grafana.zhang.com        10.111.12.239   80      32m
ingress-prometheus     <none>   prometheus.zhang.com     10.111.12.239   80      30m

八、浏览器访问

 

 

九、修改alertmanager的报警配置

1、创建alertmanger的配置文件

[root@master test]# cat  alertmanger_config.yaml 
global: 
  resolve_timeout: 5m #处理超时时间，默认为5min
  smtp_smarthost: 'smtp.163.com:465' # 邮箱smtp服务器代理
  smtp_from: 'xxxx@163.com' # 发送邮箱名称
  smtp_auth_username: 'xxxx@163.com' # 邮箱名称
  smtp_auth_password: 'xxxxxxxxx' #邮箱密码
  smtp_require_tls: false 
route:
  group_by: ['alertname'] # 报警分组名称
  group_wait: 10s # 最初即第一次等待多久时间发送一组警报的通知
  group_interval: 10s # 在发送新警报前的等待时间
  repeat_interval: 1m # 发送重复警报的周期
  receiver: 'email' # 发送警报的接收者的名称，以下receivers name的名称

receivers:
  - name: 'email' # 警报
    email_configs: # 邮箱配置
    - to: 'xxxxxx@163.com'  # 接收警报的email配置

inhibit_rules:
  - source_match:
      severity: 'critical'
    target_match:
      severity: 'warning'
    equal: ['alertname', 'dev', 'instance']

2、base64编码(要使用xargs设置为一行，再用sed删掉空格)

cat  alertmanger_config.yaml |base64 

3、替换secret中的alertmanager.yaml配置（用上面base64编码的替换）

 kubectl edit secret  alertmanager-mypro-prometheus-operator-alertmanager 

4、查看邮箱

十、自定义监控报警项

 1、介绍

prometheus-operator可以使用PrometheusRule来动态的添加自定义监控项

2、查看prometheus-operator项目中Prometheus的标签选择器

kubectl get  prometheus mypro-prometheus-operator-prometheus  -o jsonpath={".spec.ruleSelector"};echo 

3、创建自定义的PrometheusRule监控

[root@master ~]# cat  test-PrometheusRule.yaml 
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  labels:
    app: prometheus-operator   #和Prometheus中的标签选择器中的标签，如果要自己创建一个Prometheus的配置关联到PrometheusRule的labels
    release: mypro             #和Prometheus中的标签选择器中的标签，如果要自己创建一个Prometheus的配置关联到PrometheusRule的labels
    prometheus: test-example
  name: test-load1-prometheusrule  
spec:
  groups:
  - name: test-load-1
    rules:
    - alert: test-load-1
      expr: node_load1 > 1
      for: 2m
      labels:
        team: node
      annotations:
        summary: "{{$labels.instance}}: load 1 >1"
        description: "{{$labels.instance}}: job {{$labels.job}} 测试测试 负载大于1"


#导入
kubectl  apply -f test-PrometheusRule.yaml 

　　

4、登录pod查看

[root@master ~]# kubectl exec -it   prometheus-mypro-prometheus-operator-prometheus-0  sh 
kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl kubectl exec [POD] -- [COMMAND] instead.
Defaulting container name to prometheus.
Use 'kubectl describe pod/prometheus-mypro-prometheus-operator-prometheus-0 -n default' to see all of the containers in this pod.
/prometheus $ ls /etc/prometheus/rules/prometheus-mypro-prometheus-operator-prometheus-rulefiles-0/default-test-load1-prometheusrule.yaml 
/etc/prometheus/rules/prometheus-mypro-prometheus-operator-prometheus-rulefiles-0/default-test-load1-prometheusrule.yaml
/prometheus $ cat  /etc/prometheus/rules/prometheus-mypro-prometheus-operator-prometheus-rulefiles-0/default-test-load1-prometheusrule.yaml 
groups:
- name: test-load-1
  rules:
  - alert: test-load-1
    annotations:
      description: '{{$labels.instance}}: job {{$labels.job}} 测试测试 负载大于1'
      summary: '{{$labels.instance}}: load 1 >1'
    expr: node_load1 > 1
    for: 2m
    labels:
      team: node

5、浏览器查看prometheus