Kubernetes 资源预留（二）

后续

继Kubernetes 资源预留（一），查阅相关资料及参考阿里云ACK的资源预留配置，深思熟虑后，决定采取以下配置，分为二个部分

阿里云ACK配置

# 阿里云kubelet详细配置
iZj6c1ubia3yzi7sohsjtpZ:/etc/systemd/system/kubelet.service.d# cat 10-kubeadm.conf 
[Service]
EnvironmentFile=-/etc/kubernetes/kubelet-customized-args.conf
Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf"
Environment="KUBELET_SYSTEM_PODS_ARGS=--max-pods 64 --pod-max-pids 16384 --pod-manifest-path=/etc/kubernetes/manifests"
Environment="KUBELET_NETWORK_ARGS=--network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin --dynamic-config-dir=/etc/kubernetes/kubelet-config --v=3"
Environment="KUBELET_DNS_ARGS=--enable-controller-attach-detach=true --cluster-dns=192.168.0.10         --pod-infra-container-image=registry-vpc.cn-hongkong.aliyuncs.com/acs/pause:3.2         --enable-load-reader --cluster-domain=cluster.local --cloud-provider=external         --hostname-override=cn-hongkong.172.19.1.134 --provider-id=cn-hongkong.i-j6c1ubia3yzi7sohsjtp"
Environment="KUBELET_AUTHZ_ARGS=--authorization-mode=Webhook --authentication-token-webhook=true --anonymous-auth=false --client-ca-file=/etc/kubernetes/pki/ca.crt"
Environment="KUBELET_CGROUP_ARGS=--cgroup-driver=systemd"
Environment="KUBELET_CERTIFICATE_ARGS=--tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --tls-cert-file=/var/lib/kubelet/pki/kubelet.crt --tls-private-key-file=/var/lib/kubelet/pki/kubelet.key --rotate-certificates=true --cert-dir=/var/lib/kubelet/pki"
ExecStart=
ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_SYSTEM_PODS_ARGS $KUBELET_NETWORK_ARGS $KUBELET_DNS_ARGS $KUBELET_AUTHZ_ARGS $KUBELET_CGROUP_ARGS $KUBELET_CERTIFICATE_ARGS $KUBELET_EXTRA_ARGS $KUBELET_CUSTOMIZED_ARGS

# 阿里云集群节点资源预留配置
iZj6c1ubia3yzi7sohsjtpZ:/etc/systemd/system/kubelet.service.d# curl -sk https://172.19.1.134:6443/api/v1/nodes/cn-hongkong.172.19.1.134/proxy/configz | jq -r
{
  "kubeletconfig": {
    "staticPodPath": "/etc/kubernetes/manifests",
    "syncFrequency": "1m0s",
    "fileCheckFrequency": "20s",
    "httpCheckFrequency": "20s",
    "address": "0.0.0.0",
    "port": 10250,
    "readOnlyPort": 10255,
    "tlsCertFile": "/var/lib/kubelet/pki/kubelet.crt",
    "tlsPrivateKeyFile": "/var/lib/kubelet/pki/kubelet.key",
    "tlsCipherSuites": [
      "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256",
      "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
      "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305",
      "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384",
      "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305",
      "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384",
      "TLS_RSA_WITH_AES_256_GCM_SHA384",
      "TLS_RSA_WITH_AES_128_GCM_SHA256"
    ],
    "rotateCertificates": true,
    "authentication": {
      "x509": {
        "clientCAFile": "/etc/kubernetes/pki/ca.crt"
      },
      "webhook": {
        "enabled": true,
        "cacheTTL": "2m0s"
      },
      "anonymous": {
        "enabled": false
      }
    },
    "authorization": {
      "mode": "Webhook",
      "webhook": {
        "cacheAuthorizedTTL": "5m0s",
        "cacheUnauthorizedTTL": "30s"
      }
    },
    "registryPullQPS": 5,
    "registryBurst": 10,
    "eventRecordQPS": 5,
    "eventBurst": 10,
    "enableDebuggingHandlers": true,
    "healthzPort": 10248,
    "healthzBindAddress": "127.0.0.1",
    "oomScoreAdj": -999,
    "clusterDomain": "cluster.local",
    "clusterDNS": [
      "192.168.0.10"
    ],
    "streamingConnectionIdleTimeout": "4h0m0s",
    "nodeStatusUpdateFrequency": "10s",
    "nodeStatusReportFrequency": "5m0s",
    "nodeLeaseDurationSeconds": 40,
    "imageMinimumGCAge": "2m0s",
    "imageGCHighThresholdPercent": 85,
    "imageGCLowThresholdPercent": 80,
    "volumeStatsAggPeriod": "1m0s",
    "cgroupsPerQOS": true,
    "cgroupDriver": "systemd",
    "cpuManagerPolicy": "none",
    "cpuManagerReconcilePeriod": "10s",
    "topologyManagerPolicy": "none",
    "runtimeRequestTimeout": "2m0s",
    "hairpinMode": "promiscuous-bridge",
    "maxPods": 64,
    "podPidsLimit": 16384,
    "resolvConf": "/etc/resolv.conf",
    "cpuCFSQuota": true,
    "cpuCFSQuotaPeriod": "100ms",
    "maxOpenFiles": 1000000,
    "contentType": "application/vnd.kubernetes.protobuf",
    "kubeAPIQPS": 5,
    "kubeAPIBurst": 10,
    "serializeImagePulls": true,
    "evictionHard": {
      "imagefs.available": "15%",
      "memory.available": "300Mi",
      "nodefs.available": "10%",
      "nodefs.inodesFree": "5%"
    },
    "evictionPressureTransitionPeriod": "5m0s",
    "enableControllerAttachDetach": true,
    "makeIPTablesUtilChains": true,
    "iptablesMasqueradeBit": 14,
    "iptablesDropBit": 15,
    "failSwapOn": true,
    "containerLogMaxSize": "10Mi",
    "containerLogMaxFiles": 5,
    "configMapAndSecretChangeDetectionStrategy": "Watch",
    "systemReserved": {
      "memory": "300Mi",
      "pid": "10000"
    },
    "kubeReserved": {
      "memory": "400Mi",
      "pid": "10000"
    },
    "enforceNodeAllocatable": [
      "pods"
    ]
  }
}

systemReserved & kubeReserved

关于三种资源控制cpu, memroy, ephemeral-storage，该如何设置，解释如下，如果有不对请指教

1. 是否需要开启enforeceNodeAllocatable，我认为是不需要的，对于CPU/MEM大可不必强制开启，如果强制开启那么会有一个结果，就是如果预留值设置的高/低都会受影响，高则浪费；低则压力，CPU作为可压缩的资源，最重要任务就是做计算，任务结束CPU释放；MEM虽然说是一种不可压缩的资源，但是如果预留值过小，势必会触发OOM，凉凉；存储，硬盘值钱吗，合理设置报警即可
2. 参考配置如下（但是个人认为还是预留一小部分CPU/MEM即可，以免Kubernetes Pod过度分配，给OS带来负载压力）

systemReserved:
  cpu: 500m
  memory: 500Mi
kubeReserved:
  cpu: 500m
  memory: 500Mi

evictionHard

evictionHard的主要功能是，防止Kubernetes节点的资源被过度消耗，避免触发节点OOM，在这里要理解Node OOM & Pod 驱逐（前者是在资源极度缺乏下由Node触发OOM机制；后者是根据evictionHard的阈值，在触发到阈值将Pod驱逐，具体参考驱逐策略及处理方式，官方引文）

具体参考性配置如下

evictionHard:
  memory.available: "500Mi"
  nodefs.available: "10%"
  nodefs.inodesFree: "5%"
  imagefs.available: "15%"
evictionMinimumReclaim:
  memory.available: "0Mi"
  nodefs.available: "500Mi"
  imagefs.available: "2Gi"