k8s ingress使用DaemonSet部署

1. ingress Controller高可用方案

• 一般ingress Controller会以Daemonset+nodeSelector部署到几台特定node，然后将这几台挂载到公网负载均衡器对外提供服务。
  

2. 案例部署多个ingress Controller

2.1 查询nginx-ingress-controller服务

[root@k8s-master ingress]# kubectl get pod -n ingress-nginx 
NAME                                       READY   STATUS    RESTARTS   AGE
nginx-ingress-controller-5dc64b58f-ctjfd   1/1     Running   0          23h

2.2 删除已有服务

[root@k8s-master ingress]# kubectl delete deploy nginx-ingress-controller -n ingress-nginx 
deployment.apps "nginx-ingress-controller" deleted

2.3 修改ingress controller配置文件

• 给nodes打上标签

  [root@k8s-master ~]# kubectl get nodes --show-labels 
  NAME         STATUS   ROLES    AGE   VERSION   LABELS
  k8s-master   Ready    master   25h   v1.19.0   beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=k8s-master,kubernetes.io/os=linux,node-role.kubernetes.io/master=
  k8s-node1    Ready    <none>   25h   v1.19.0   beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=k8s-node1,kubernetes.io/os=linux
  k8s-node2    Ready    <none>   25h   v1.19.0   beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=k8s-node2,kubernetes.io/os=linux
  
  [root@k8s-master ~]#  kubectl label nodes k8s-node1 isIngress="isingress"
  node/k8s-node1 labeled
  
  [root@k8s-master ~]# kubectl label nodes k8s-node2 isIngress="isingress"
  node/k8s-node2 labeled
  
  [root@k8s-master ~]# kubectl get nodes --show-labels 
  NAME         STATUS   ROLES    AGE   VERSION   LABELS
  k8s-master   Ready    master   25h   v1.19.0   beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=k8s-master,kubernetes.io/os=linux,node-role.kubernetes.io/master=
  k8s-node1    Ready    <none>   25h   v1.19.0   beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,isIngress=isingress,kubernetes.io/arch=amd64,kubernetes.io/hostname=k8s-node1,kubernetes.io/os=linux
  k8s-node2    Ready    <none>   25h   v1.19.0   beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,isIngress=isingress,kubernetes.io/arch=amd64,kubernetes.io/hostname=k8s-node2,kubernetes.io/os=linux
  

• 修改内容

  # 需要修改的内容
  kind: Deployment
  
  # 需要修改的内容
        nodeSelector:
          kubernetes.io/os: linux
  #修改后的内容
  kind: DaemonSet
  
  #修改后的节点nodeSelector节点
        nodeSelector:
          isIngress: isingress
  

• 修改后的配置文件内容

  [root@k8s-master ~]# vim ingress_controller.yaml 
  [root@k8s-master ~]# cat ingress_controller.yaml 
  apiVersion: v1
  kind: Namespace
  metadata:
    name: ingress-nginx
    labels:
      app.kubernetes.io/name: ingress-nginx
      app.kubernetes.io/part-of: ingress-nginx
  
  ---
  
  kind: ConfigMap
  apiVersion: v1
  metadata:
    name: nginx-configuration
    namespace: ingress-nginx
    labels:
      app.kubernetes.io/name: ingress-nginx
      app.kubernetes.io/part-of: ingress-nginx
  
  ---
  kind: ConfigMap
  apiVersion: v1
  metadata:
    name: tcp-services
    namespace: ingress-nginx
    labels:
      app.kubernetes.io/name: ingress-nginx
      app.kubernetes.io/part-of: ingress-nginx
  
  ---
  kind: ConfigMap
  apiVersion: v1
  metadata:
    name: udp-services
    namespace: ingress-nginx
    labels:
      app.kubernetes.io/name: ingress-nginx
      app.kubernetes.io/part-of: ingress-nginx
  
  ---
  apiVersion: v1
  kind: ServiceAccount
  metadata:
    name: nginx-ingress-serviceaccount
    namespace: ingress-nginx
    labels:
      app.kubernetes.io/name: ingress-nginx
      app.kubernetes.io/part-of: ingress-nginx
  
  ---
  apiVersion: rbac.authorization.k8s.io/v1
  kind: ClusterRole
  metadata:
    name: nginx-ingress-clusterrole
    labels:
      app.kubernetes.io/name: ingress-nginx
      app.kubernetes.io/part-of: ingress-nginx
  rules:
    - apiGroups:
        - ""
      resources:
        - configmaps
        - endpoints
        - nodes
        - pods
        - secrets
      verbs:
        - list
        - watch
    - apiGroups:
        - ""
      resources:
        - nodes
      verbs:
        - get
    - apiGroups:
        - ""
      resources:
        - services
      verbs:
        - get
        - list
        - watch
    - apiGroups:
        - ""
      resources:
        - events
      verbs:
        - create
        - patch
    - apiGroups:
        - "extensions"
        - "networking.k8s.io"
      resources:
        - ingresses
      verbs:
        - get
        - list
        - watch
    - apiGroups:
        - "extensions"
        - "networking.k8s.io"
      resources:
        - ingresses/status
      verbs:
        - update
  
  ---
  apiVersion: rbac.authorization.k8s.io/v1
  kind: Role
  metadata:
    name: nginx-ingress-role
    namespace: ingress-nginx
    labels:
      app.kubernetes.io/name: ingress-nginx
      app.kubernetes.io/part-of: ingress-nginx
  rules:
    - apiGroups:
        - ""
      resources:
        - configmaps
        - pods
        - secrets
        - namespaces
      verbs:
        - get
    - apiGroups:
        - ""
      resources:
        - configmaps
      resourceNames:
        # Defaults to "<election-id>-<ingress-class>"
        # Here: "<ingress-controller-leader>-<nginx>"
        # This has to be adapted if you change either parameter
        # when launching the nginx-ingress-controller.
        - "ingress-controller-leader-nginx"
      verbs:
        - get
        - update
    - apiGroups:
        - ""
      resources:
        - configmaps
      verbs:
        - create
    - apiGroups:
        - ""
      resources:
        - endpoints
      verbs:
        - get
  
  ---
  apiVersion: rbac.authorization.k8s.io/v1
  kind: RoleBinding
  metadata:
    name: nginx-ingress-role-nisa-binding
    namespace: ingress-nginx
    labels:
      app.kubernetes.io/name: ingress-nginx
      app.kubernetes.io/part-of: ingress-nginx
  roleRef:
    apiGroup: rbac.authorization.k8s.io
    kind: Role
    name: nginx-ingress-role
  subjects:
    - kind: ServiceAccount
      name: nginx-ingress-serviceaccount
      namespace: ingress-nginx
  
  ---
  apiVersion: rbac.authorization.k8s.io/v1
  kind: ClusterRoleBinding
  metadata:
    name: nginx-ingress-clusterrole-nisa-binding
    labels:
      app.kubernetes.io/name: ingress-nginx
      app.kubernetes.io/part-of: ingress-nginx
  roleRef:
    apiGroup: rbac.authorization.k8s.io
    kind: ClusterRole
    name: nginx-ingress-clusterrole
  subjects:
    - kind: ServiceAccount
      name: nginx-ingress-serviceaccount
      namespace: ingress-nginx
  
  ---
  
  apiVersion: apps/v1
  # 修改为daemonSet方式部署，保证每个节点都部署一个pod
  kind: DaemonSet
  metadata:
    name: nginx-ingress-controller
    namespace: ingress-nginx
    labels:
      app.kubernetes.io/name: ingress-nginx
      app.kubernetes.io/part-of: ingress-nginx
  spec:
    selector:
      matchLabels:
        app.kubernetes.io/name: ingress-nginx
        app.kubernetes.io/part-of: ingress-nginx
    template:
      metadata:
        labels:
          app.kubernetes.io/name: ingress-nginx
          app.kubernetes.io/part-of: ingress-nginx
        annotations:
          prometheus.io/port: "10254"
          prometheus.io/scrape: "true"
      spec:
        # wait up to five minutes for the drain of connections
        terminationGracePeriodSeconds: 300
        serviceAccountName: nginx-ingress-serviceaccount
        # 修改ingress controller指定分配到标签的那个节点运行
        nodeSelector:
          isIngress: isingress
        hostNetwork: true
        containers:
          - name: nginx-ingress-controller
            image: shichao01/nginx-ingress-controller:0.30.0
            args:
              - /nginx-ingress-controller
              - --configmap=$(POD_NAMESPACE)/nginx-configuration
              - --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services
              - --udp-services-configmap=$(POD_NAMESPACE)/udp-services
              - --publish-service=$(POD_NAMESPACE)/ingress-nginx
              - --annotations-prefix=nginx.ingress.kubernetes.io
            securityContext:
              allowPrivilegeEscalation: true
              capabilities:
                drop:
                  - ALL
                add:
                  - NET_BIND_SERVICE
              # www-data -> 101
              runAsUser: 101
            env:
              - name: POD_NAME
                valueFrom:
                  fieldRef:
                    fieldPath: metadata.name
              - name: POD_NAMESPACE
                valueFrom:
                  fieldRef:
                    fieldPath: metadata.namespace
            ports:
              - name: http
                containerPort: 80
                protocol: TCP
              - name: https
                containerPort: 443
                protocol: TCP
            livenessProbe:
              failureThreshold: 3
              httpGet:
                path: /healthz
                port: 10254
                scheme: HTTP
              initialDelaySeconds: 10
              periodSeconds: 10
              successThreshold: 1
              timeoutSeconds: 10
            readinessProbe:
              failureThreshold: 3
              httpGet:
                path: /healthz
                port: 10254
                scheme: HTTP
              periodSeconds: 10
              successThreshold: 1
              timeoutSeconds: 10
            lifecycle:
              preStop:
                exec:
                  command:
                    - /wait-shutdown
  
  ---
  
  apiVersion: v1
  kind: LimitRange
  metadata:
    name: ingress-nginx
    namespace: ingress-nginx
    labels:
      app.kubernetes.io/name: ingress-nginx
      app.kubernetes.io/part-of: ingress-nginx
  spec:
    limits:
    - min:
        memory: 90Mi
        cpu: 100m
      type: Container
  

• 启动服务

  [root@k8s-master ~]# kubectl apply -f ingress_controller.yaml 
  namespace/ingress-nginx unchanged
  configmap/nginx-configuration unchanged
  configmap/tcp-services unchanged
  configmap/udp-services unchanged
  serviceaccount/nginx-ingress-serviceaccount unchanged
  clusterrole.rbac.authorization.k8s.io/nginx-ingress-clusterrole configured
  role.rbac.authorization.k8s.io/nginx-ingress-role configured
  rolebinding.rbac.authorization.k8s.io/nginx-ingress-role-nisa-binding configured
  clusterrolebinding.rbac.authorization.k8s.io/nginx-ingress-clusterrole-nisa-binding configured
  daemonset.apps/nginx-ingress-controller created
  limitrange/ingress-nginx configured
  

• 验证服务

  [root@k8s-master ~]# kubectl get pod -n ingress-nginx -o wide
  NAME                             READY   STATUS    RESTARTS   AGE     IP              NODE        NOMINATED NODE   READINESS GATES
  nginx-ingress-controller-k9cf9   1/1     Running   0          2m52s   192.168.0.202   k8s-node1   <none>           <none>
  nginx-ingress-controller-kvxr8   1/1     Running   0          2m52s   192.168.0.203   k8s-node2   <none>           <none>