配置各个节点/etc/hosts
[root@host-10-10-18-42 etcd]# cat /etc/hosts 127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 ::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 etcd1 10.10.18.42 etcd2 10.10.18.43 etcd3 10.10.18.44
| etcd1 |
10.10.18.42 |
|
|
10.10.18.43 |
|
|
10.10.18.44 |
配置目录
mkdir /data/k8s/etcd/{data,wal} -p
chown -R etcd.etcd /data/k8s/etcd
etcd1
[root@host-10-10-18-42 etcd]# cat etcd.conf ETCD_DATA_DIR="/data/k8s/etcd/data" ETCD_WAL_DIR="/data/k8s/etcd/wal" ETCD_LISTEN_PEER_URLS="http://10.10.18.42:2380" ETCD_LISTEN_CLIENT_URLS="http://10.10.18.42:2379" ETCD_MAX_SNAPSHOTS="5" ETCD_MAX_WALS="5" ETCD_NAME="etcd1" ETCD_SNAPSHOT_COUNT="100000" ETCD_HEARTBEAT_INTERVAL="100" ETCD_ELECTION_TIMEOUT="1000" ETCD_INITIAL_ADVERTISE_PEER_URLS="http://10.10.18.42:2380" ETCD_ADVERTISE_CLIENT_URLS="http://10.10.18.42:2379" ETCD_INITIAL_CLUSTER="etcd1=http://10.10.18.42:2380,etcd2=http://10.10.18.43:2380,etcd3=http://10.10.18.44:2380" ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster" ETCD_INITIAL_CLUSTER_STATE="new"
etcd2
[root@host-10-10-18-43 etcd]# cat etcd.conf ETCD_DATA_DIR="/data/k8s/etcd/data" ETCD_WAL_DIR="/data/k8s/etcd/wal" ETCD_LISTEN_PEER_URLS="http://10.10.18.43:2380" ETCD_LISTEN_CLIENT_URLS="http://10.10.18.43:2379" ETCD_MAX_SNAPSHOTS="5" ETCD_MAX_WALS="5" ETCD_NAME="etcd2" ETCD_SNAPSHOT_COUNT="100000" ETCD_HEARTBEAT_INTERVAL="100" ETCD_ELECTION_TIMEOUT="1000" ETCD_INITIAL_ADVERTISE_PEER_URLS="http://10.10.18.43:2380" ETCD_ADVERTISE_CLIENT_URLS="http://10.10.18.43:2379" ETCD_INITIAL_CLUSTER="etcd1=http://10.10.18.42:2380,etcd2=http://10.10.18.43:2380,etcd3=http://10.10.18.44:2380" ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster" ETCD_INITIAL_CLUSTER_STATE="new"
etcd3
[root@host-10-10-18-44 etcd]# cat etcd.conf ETCD_DATA_DIR="/data/k8s/etcd/data" ETCD_WAL_DIR="/data/k8s/etcd/wal" ETCD_LISTEN_PEER_URLS="http://10.10.18.44:2380" ETCD_LISTEN_CLIENT_URLS="http://10.10.18.44:2379" ETCD_MAX_SNAPSHOTS="5" ETCD_MAX_WALS="5" ETCD_NAME="etcd3" ETCD_SNAPSHOT_COUNT="100000" ETCD_HEARTBEAT_INTERVAL="100" ETCD_ELECTION_TIMEOUT="1000" ETCD_INITIAL_ADVERTISE_PEER_URLS="http://10.10.18.44:2380" ETCD_ADVERTISE_CLIENT_URLS="http://10.10.18.44:2379" ETCD_INITIAL_CLUSTER="etcd1=http://10.10.18.42:2380,etcd2=http://10.10.18.43:2380,etcd3=http://10.10.18.44:2380" ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster" ETCD_INITIAL_CLUSTER_STATE="new"
Jun 29 11:28:03 host-10-10-18-43 etcd[4059]: health check for peer 1829ea2c82ecd13e could not connect: dial tcp 10.10.18.42:2380: i/o timeout (prober "ROUND_TRIPPER_RAFT_MESSAGE") Jun 29 11:28:03 host-10-10-18-43 etcd[4059]: health check for peer 1829ea2c82ecd13e could not connect: dial tcp 10.10.18.42:2380: i/o timeout (prober "ROUND_TRIPPER_SNAPSHOT") Jun 29 11:28:03 host-10-10-18-43 etcd[4059]: health check for peer fe3b541533812c5d could not connect: dial tcp 10.10.18.44:2380: i/o timeout (prober "ROUND_TRIPPER_RAFT_MESSAGE") Jun 29 11:28:03 host-10-10-18-43 etcd[4059]: health check for peer fe3b541533812c5d could not connect: dial tcp 10.10.18.44:2380: i/o timeout (prober "ROUND_TRIPPER_SNAPSHOT")
root@ubuntu:~/bibili# telnet 10.10.18.44 2380 Trying 10.10.18.44... telnet: Unable to connect to remote host: No route to host root@ubuntu:~/bibili#
关闭CentOS7防火墙
# 查看防火墙状态 firewall-cmd --state # 停止firewall systemctl stop firewalld.service # 禁止firewall开机启动 systemctl disable firewalld.service
关闭SELINUX
# 编辑SELINUX文件 vim /etc/selinux/config # 将SELINUX=enforcing改为SELINUX=disabled
root@ubuntu:~/bibili# telnet 10.10.18.44 2380 Trying 10.10.18.44... Connected to 10.10.18.44. Escape character is '^]'. ^C^C^CConnection closed by foreign host.
root@ubuntu:~/etcd-v3.5.0-linux-arm64# ETCDCTL_API=3 etcdctl --endpoints=http://10.10.18.42:2379,http://10.10.18.43:2379,http://10.10.18.44:2379 endpoint health http://10.10.18.43:2379 is healthy: successfully committed proposal: took = 2.311413ms http://10.10.18.42:2379 is healthy: successfully committed proposal: took = 4.239303ms http://10.10.18.44:2379 is healthy: successfully committed proposal: took = 4.742326ms root@ubuntu:~/etcd-v3.5.0-linux-arm64#
root@ubuntu:~/etcd-v3.5.0-linux-arm64# ETCDCTL_API=3 ./etcdctl --endpoints=http://10.10.18.42:2379,http://10.10.18.43:2379,http://10.10.18.44:2379 member list 1829ea2c82ecd13e, started, etcd1, http://10.10.18.42:2380, http://10.10.18.42:2379, false 19ddebfcb3e299fd, started, etcd2, http://10.10.18.43:2380, http://10.10.18.43:2379, false fe3b541533812c5d, started, etcd3, http://10.10.18.44:2380, http://10.10.18.44:2379, false
root@ubuntu:~/etcd-v3.5.0-linux-arm64# ETCDCTL_API=3 ./etcdctl -w table --endpoints=http://10.10.18.42:2379,http://10.10.18.43:2379,http://10.10.18.44:2379 endpoint status +-------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+ | ENDPOINT | ID | VERSION | DB SIZE | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX | ERRORS | +-------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+ | http://10.10.18.42:2379 | 1829ea2c82ecd13e | 3.3.11 | 328 kB | true | false | 339 | 17 | 0 | | | http://10.10.18.43:2379 | 19ddebfcb3e299fd | 3.3.11 | 328 kB | false | false | 339 | 17 | 0 | | | http://10.10.18.44:2379 | fe3b541533812c5d | 3.3.11 | 328 kB | false | false | 339 | 17 | 0 | | +-------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+ root@ubuntu:~/etcd-v3.5.0-linux-arm64#
root@ubuntu:~/etcd-v3.5.0-linux-arm64# ENDPOINTS=http://10.10.18.42:2379,http://10.10.18.43:2379,http://10.10.18.44:2379 root@ubuntu:~/etcd-v3.5.0-linux-arm64# ETCDCTL_API=3 ./etcdctl -w table --endpoints=$ENDPOINTS endpoint status +-------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+ | ENDPOINT | ID | VERSION | DB SIZE | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX | ERRORS | +-------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+ | http://10.10.18.42:2379 | 1829ea2c82ecd13e | 3.3.11 | 328 kB | true | false | 339 | 17 | 0 | | | http://10.10.18.43:2379 | 19ddebfcb3e299fd | 3.3.11 | 328 kB | false | false | 339 | 17 | 0 | | | http://10.10.18.44:2379 | fe3b541533812c5d | 3.3.11 | 328 kB | false | false | 339 | 17 | 0 | | +-------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+ root@ubuntu:~/etcd-v3.5.0-linux-arm64#
root@ubuntu:~/etcd-v3.5.0-linux-arm64# ETCDCTL_API=3 ./etcdctl --endpoints=$ENDPOINTS put test "helloworld" OK root@ubuntu:~/etcd-v3.5.0-linux-arm64# ETCDCTL_API=3 ./etcdctl --endpoints=$ENDPOINTS get test test helloworld root@ubuntu:~/etcd-v3.5.0-linux-arm64#
[root@host-10-10-18-42 etcd]# tree /data/k8s/etcd/ /data/k8s/etcd/ |-- data | `-- member | `-- snap | `-- db `-- wal |-- 0000000000000000-0000000000000000.wal `-- 0.tmp 4 directories, 3 files
root@ubuntu:~/etcd-v3.5.0-linux-arm64# ETCDCTL_API=3 ./etcdctl --endpoints=$ENDPOINTS snapshot save mysnapshot.db Error: snapshot must be requested to one selected node, not multiple [http://10.10.18.42:2379 http://10.10.18.43:2379 http://10.10.18.44:2379] root@ubuntu:~/etcd-v3.5.0-linux-arm64# ETCDCTL_API=3 ./etcdctl --endpoints=$ENDPOINTS snapshot status mysnapshot.db -w json Deprecated: Use `etcdutl snapshot status` instead. Error: stat mysnapshot.db: no such file or directory root@ubuntu:~/etcd-v3.5.0-linux-arm64# ETCDCTL_API=3 ./etcdctl --endpoints=http://10.10.18.43:2379 snapshot status mysnapshot.db -w json Deprecated: Use `etcdutl snapshot status` instead. Error: stat mysnapshot.db: no such file or directory root@ubuntu:~/etcd-v3.5.0-linux-arm64# ETCDCTL_API=3 ./etcdctl --endpoints=http://10.10.18.43:2379 snapshot save mysnapshot.db {"level":"info","ts":1624938894.0369105,"caller":"snapshot/v3_snapshot.go:68","msg":"created temporary db file","path":"mysnapshot.db.part"} {"level":"info","ts":1624938894.0386374,"logger":"client","caller":"v3/maintenance.go:211","msg":"opened snapshot stream; downloading"} {"level":"info","ts":1624938894.0386932,"caller":"snapshot/v3_snapshot.go:76","msg":"fetching snapshot","endpoint":"http://10.10.18.43:2379"} {"level":"info","ts":1624938894.0599878,"logger":"client","caller":"v3/maintenance.go:219","msg":"completed snapshot read; closing"} {"level":"info","ts":1624938894.0606616,"caller":"snapshot/v3_snapshot.go:91","msg":"fetched snapshot","endpoint":"http://10.10.18.43:2379","size":"328 kB","took":"now"} {"level":"info","ts":1624938894.0607412,"caller":"snapshot/v3_snapshot.go:100","msg":"saved","path":"mysnapshot.db"} Snapshot saved at mysnapshot.db root@ubuntu:~/etcd-v3.5.0-linux-arm64# ETCDCTL_API=3 ./etcdctl snapshot status mysnapshot.db -w json Deprecated: Use `etcdutl snapshot status` instead. {"hash":3787458990,"revision":2,"totalKey":7,"totalSize":327680} root@ubuntu:~/etcd-v3.5.0-linux-arm64#
etcd 秘钥
ETCDCTL_API=3 etcdctl --endpoints=https://127.0.0.1:2379
--cacert=<trusted-ca-file> --cert=<cert-file> --key=<key-file> --cacert="" verify certificates of TLS-enabled secure servers using this CA bundle --cert="" identify secure client using this TLS certificate file --key="" identify secure client using this TLS key file --endpoints=[127.0.0.1:2379] gRPC endpoints
root@ubuntu:~/etcd-v3.5.0-linux-arm64# ls /etc/kubernetes/pki/etcd/ ca.crt ca.key healthcheck-client.crt healthcheck-client.key peer.crt peer.key server.crt server.key root@ubuntu:~/etcd-v3.5.0-linux-arm64# ETCDCTL_API=3 ./etcdctl --cacert=/etc/kubernetes/pki/etcd/ca.crt --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key --endpoints=$ENDPOINTS member list 1829ea2c82ecd13e, started, etcd1, http://10.10.18.42:2380, http://10.10.18.42:2379, false 19ddebfcb3e299fd, started, etcd2, http://10.10.18.43:2380, http://10.10.18.43:2379, false fe3b541533812c5d, started, etcd3, http://10.10.18.44:2380, http://10.10.18.44:2379, false root@ubuntu:~/etcd-v3.5.0-linux-arm64#
k8s 和etcd
TLS 认证文件
需要为 etcd 集群创建加密通信的 TLS 证书，这里复用以前创建的 kubernetes 证书
root@ubuntu:/etc# ps -elf | grep etcd 4 S root 7969 7939 2 80 0 - 2672731 futex_ Jun18 ? 05:35:58 etcd --advertise-client-urls=https://10.10.16.82:2379 --cert-file=/etc/kubernetes/pki/etcd/server.crt --client-cert-auth=true --data-dir=/var/lib/etcd --initial-advertise-peer-urls=https://10.10.16.82:2380 --initial-cluster=ubuntu=https://10.10.16.82:2380 --key-file=/etc/kubernetes/pki/etcd/server.key --listen-client-urls=https://127.0.0.1:2379,https://10.10.16.82:2379 --listen-metrics-urls=http://127.0.0.1:2381 --listen-peer-urls=https://10.10.16.82:2380 --name=ubuntu --peer-cert-file=/etc/kubernetes/pki/etcd/peer.crt --peer-client-cert-auth=true --peer-key-file=/etc/kubernetes/pki/etcd/peer.key --peer-trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt --snapshot-count=10000 --trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt
[root@host-10-10-18-42 system]# curl --cacert /etc/kubernetes/pki/etcd/ca.crt --cert /etc/kubernetes/pki/etcd/server.crt --key /etc/kubernetes/pki/etcd/server.key https://10.10.16.82:2379/version {"etcdserver":"3.4.3","etcdcluster":"3.4.0"}
[root@host-10-10-18-42 system]# ls /etc/kubernetes/pki/etcd/ ca.crt ca.key healthcheck-client.crt healthcheck-client.key peer.crt peer.key server.crt server.key [root@host-10-10-18-42 system]#
[root@host-10-10-18-42 etc]# systemctl status etcd.service -l ● etcd.service - Etcd Server Loaded: loaded (/usr/lib/systemd/system/etcd.service; disabled; vendor preset: disabled) Active: active (running) since Tue 2021-06-29 11:36:30 CST; 3h 11min ago Main PID: 4306 (etcd) CGroup: /system.slice/etcd.service └─4306 /usr/bin/etcd --name=etcd1 --data-dir=/data/k8s/etcd/data --listen-client-urls=http://10.10.18.42:2379 Jun 29 12:10:56 host-10-10-18-42 etcd[4306]: failed to send out heartbeat on time (exceeded the 100ms timeout for 31.49688ms) Jun 29 12:10:56 host-10-10-18-42 etcd[4306]: server is likely overloaded Jun 29 12:38:51 host-10-10-18-42 etcd[4306]: failed to send out heartbeat on time (exceeded the 100ms timeout for 36.45658ms) Jun 29 12:38:51 host-10-10-18-42 etcd[4306]: server is likely overloaded Jun 29 12:38:51 host-10-10-18-42 etcd[4306]: failed to send out heartbeat on time (exceeded the 100ms timeout for 217.69444ms) Jun 29 12:38:51 host-10-10-18-42 etcd[4306]: server is likely overloaded Jun 29 14:00:07 host-10-10-18-42 etcd[4306]: failed to send out heartbeat on time (exceeded the 100ms timeout for 5.25766ms) Jun 29 14:00:07 host-10-10-18-42 etcd[4306]: server is likely overloaded Jun 29 14:00:07 host-10-10-18-42 etcd[4306]: failed to send out heartbeat on time (exceeded the 100ms timeout for 33.283ms) Jun 29 14:00:07 host-10-10-18-42 etcd[4306]: server is likely overloaded
the server is already initialized as member before, starting as etcd member
Jun 29 15:46:01 host-10-10-18-42 etcd[18666]: the server is already initialized as member before, starting as etcd member... Jun 29 15:46:01 host-10-10-18-42 etcd[18666]: peerTLS: cert = /etc/kubernetes/pki/etcd/peer.crt, key = /etc/kubernetes/pki/etcd/peer.key, ca = , trusted-ca = /etc/kubernetes/pki/etcd/ca.crt, client-cert-auth = true, crl-file = Jun 29 15:46:01 host-10-10-18-42 etcd[18666]: The scheme of peer url http://10.10.18.42:2380 is HTTP while peer key/cert files are presented. Ignored peer key/cert files. Jun 29 15:46:01 host-10-10-18-42 etcd[18666]: The scheme of peer url http://10.10.18.42:2380 is HTTP while client cert auth (--peer-client-cert-auth) is enabled. Ignored client cert auth for this url. Jun 29 15:46:01 host-10-10-18-42 etcd[18666]: listening for peers on http://10.10.18.42:2380 Jun 29 15:46:01 host-10-10-18-42 etcd[18666]: The scheme of client url http://10.10.18.42:2379 is HTTP while peer key/cert files are presented. Ignored key/cert files. Jun 29 15:46:01 host-10-10-18-42 etcd[18666]: The scheme of client url http://10.10.18.42:2379 is HTTP while client cert auth (--client-cert-auth) is enabled. Ignored client cert auth for this url. Jun 29 15:46:01 host-10-10-18-42 etcd[18666]: listening for client requests on 10.10.18.42:2379 Jun 29 15:46:01 host-10-10-18-42 etcd[18666]: open /etc/kubernetes/pki/etcd/peer.key: permission denied Jun 29 15:46:01 host-10-10-18-42 systemd[1]: etcd.service: main process exited, code=exited, status=1/FAILURE Jun 29 15:46:01 host-10-10-18-42 systemd[1]: Failed to start Etcd Server.
[root@host-10-10-18-42 system]# chown -R etcd.etcd /etc/kubernetes/pki/etcd
/usr/lib/systemd/system/etcd.service
添加秘钥
[root@host-10-10-18-42 system]# cat etcd.service [Unit] Description=Etcd Server After=network.target After=network-online.target Wants=network-online.target [Service] Type=notify WorkingDirectory=/var/lib/etcd/ EnvironmentFile=-/etc/etcd/etcd.conf User=etcd # set GOMAXPROCS to number of processors ExecStart=/bin/bash -c "GOMAXPROCS=$(nproc) /usr/bin/etcd --name="${ETCD_NAME}" --data-dir="${ETCD_DATA_DIR}" --listen-client-urls="${ETCD_LISTEN_CLIENT_URLS}" --cert-file=/etc/kubernetes/pki/etcd/server.crt --client-cert-auth=true --key-file=/etc/kubernetes/pki/etcd/server.key --peer-key-file=/etc/kubernetes/pki/etcd/peer.key --peer-trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt --trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt --peer-cert-file=/etc/kubernetes/pki/etcd/peer.crt --peer-client-cert-auth=true" Restart=on-failure LimitNOFILE=65536 [Install] WantedBy=multi-user.target
[root@host-10-10-18-42 system]# systemctl daemon-reload [root@host-10-10-18-42 system]# systemctl restart etcd [root@host-10-10-18-42 system]#
root@ubuntu:~/etcd-v3.5.0-linux-arm64# ETCDCTL_API=3 ./etcdctl -w table --endpoints=$ENDPOINTS endpoint status +-------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+ | ENDPOINT | ID | VERSION | DB SIZE | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX | ERRORS | +-------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+ | http://10.10.18.42:2379 | 1829ea2c82ecd13e | 3.3.11 | 262 kB | false | false | 270 | 13 | 0 | | | http://10.10.18.43:2379 | 19ddebfcb3e299fd | 3.3.11 | 262 kB | false | false | 270 | 13 | 0 | | | http://10.10.18.44:2379 | fe3b541533812c5d | 3.3.11 | 262 kB | true | false | 270 | 13 | 0 | | +-------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+ root@ubuntu:~/etcd-v3.5.0-linux-arm64#
root@ubuntu:~/etcd-v3.5.0-linux-arm64# ETCDCTL_API=3 ./etcdctl -w table --endpoints=$ENDPOINTS endpoint health +-------------------------+--------+-------------+-------+ | ENDPOINT | HEALTH | TOOK | ERROR | +-------------------------+--------+-------------+-------+ | http://10.10.18.44:2379 | true | 14.214315ms | | | http://10.10.18.43:2379 | true | 17.301696ms | | | http://10.10.18.42:2379 | true | 14.207596ms | | +-------------------------+--------+-------------+-------+ root@ubuntu:~/etcd-v3.5.0-linux-arm64# ETCDCTL_API=3 ./etcdctl -w table --endpoints=$ENDPOINTS member list +------------------+---------+-------+-------------------------+-------------------------+------------+ | ID | STATUS | NAME | PEER ADDRS | CLIENT ADDRS | IS LEARNER | +------------------+---------+-------+-------------------------+-------------------------+------------+ | 1829ea2c82ecd13e | started | etcd1 | http://10.10.18.42:2380 | http://10.10.18.42:2379 | false | | 19ddebfcb3e299fd | started | etcd2 | http://10.10.18.43:2380 | http://10.10.18.43:2379 | false | | fe3b541533812c5d | started | etcd3 | http://10.10.18.44:2380 | http://10.10.18.44:2379 | false | +------------------+---------+-------+-------------------------+-------------------------+------------+ root@ubuntu:~/etcd-v3.5.0-linux-arm64#
k8s controller
To start using your cluster, you need to run the following as a regular user: mkdir -p $HOME/.kube sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config sudo chown $(id -u):$(id -g) $HOME/.kube/config You should now deploy a pod network to the cluster. Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at: https://kubernetes.io/docs/concepts/cluster-administration/addons/ Then you can join any number of worker nodes by running the following on each as root: kubeadm join 10.10.18.46:6443 --token pbje64.ffl4ms0ymvjhwu52 --discovery-token-ca-cert-hash sha256:037f81a4c3dab193f50af44af460032172f7b8a700109c9ebebcc731728b165f [root@host-10-10-18-46 ~]# mkdir -p $HOME/.kube [root@host-10-10-18-46 ~]# cp -i /etc/kubernetes/admin.conf $HOME/.kube/config [root@host-10-10-18-46 ~]# chown $(id -u):$(id -g) $HOME/.kube/config [root@host-10-10-18-46 ~]# kubeadm config print init-defaults > kubeadm-init.yaml.yaml W0629 17:29:28.721072 20178 configset.go:202] WARNING: kubeadm cannot validate component configs for API groups [kubelet.config.k8s.io kubeproxy.config.k8s.io] [root@host-10-10-18-46 ~]#
[root@host-10-10-18-46 ~]# ls anaconda-ks.cfg k8s.init kubeadm-init.yaml.yaml [root@host-10-10-18-46 ~]# kubeadm init --config=kubeadm-init.yaml.yaml W0629 17:46:59.845088 14974 strict.go:54] error unmarshaling configuration schema.GroupVersionKind{Group:"kubeadm.k8s.io", Version:"v1beta2", Kind:"ClusterConfiguration"}: error converting YAML to JSON: yaml: unmarshal errors: line 15: key "imageRepository" already set in map line 18: key "apiServer" already set in map line 24: key "etcd" already set in map W0629 17:46:59.847076 14974 configset.go:202] WARNING: kubeadm cannot validate component configs for API groups [kubelet.config.k8s.io kubeproxy.config.k8s.io] [init] Using Kubernetes version: v1.18.0 [preflight] Running pre-flight checks [WARNING Service-Docker]: docker service is not enabled, please run 'systemctl enable docker.service' [WARNING IsDockerSystemdCheck]: detected "cgroupfs" as the Docker cgroup driver. The recommended driver is "systemd". Please follow the guide at https://kubernetes.io/docs/setup/cri/ [WARNING SystemVerification]: this Docker version is not on the list of validated versions: 20.10.7. Latest validated version: 19.03 error execution phase preflight: [preflight] Some fatal errors occurred: [ERROR Port-6443]: Port 6443 is in use [ERROR Port-10259]: Port 10259 is in use [ERROR Port-10257]: Port 10257 is in use [ERROR FileAvailable--etc-kubernetes-manifests-kube-apiserver.yaml]: /etc/kubernetes/manifests/kube-apiserver.yaml already exists [ERROR FileAvailable--etc-kubernetes-manifests-kube-controller-manager.yaml]: /etc/kubernetes/manifests/kube-controller-manager.yaml already exists [ERROR FileAvailable--etc-kubernetes-manifests-kube-scheduler.yaml]: /etc/kubernetes/manifests/kube-scheduler.yaml already exists [ERROR FileAvailable--etc-kubernetes-manifests-etcd.yaml]: /etc/kubernetes/manifests/etcd.yaml already exists [ERROR Port-10250]: Port 10250 is in use [ERROR ExternalEtcdVersion]: Get https://10.10.18.42:2379/version: EOF [ERROR ExternalEtcdVersion]: Get https://10.10.18.43:2379/version: EOF [ERROR ExternalEtcdVersion]: Get https://10.10.18.44:2379/version: EOF [preflight] If you know what you are doing, you can make a check non-fatal with `--ignore-preflight-errors=...` To see the stack trace of this error execute with --v=5 or higher
[root@host-10-10-18-46 ~]# vi kubeadm-init.yaml.yaml apiVersion: kubeadm.k8s.io/v1beta2 bootstrapTokens: - groups: - system:bootstrappers:kubeadm:default-node-token token: abcdef.0123456789abcdef ttl: 24h0m0s usages: - signing - authentication kind: InitConfiguration localAPIEndpoint: advertiseAddress: 1.2.3.4 bindPort: 6443 nodeRegistration: criSocket: /var/run/dockershim.sock name: host-10-10-18-46 taints: - effect: NoSchedule key: node-role.kubernetes.io/master --- apiServer: timeoutForControlPlane: 4m0s apiVersion: kubeadm.k8s.io/v1beta2 certificatesDir: /etc/kubernetes/pki clusterName: kubernetes controllerManager: {} dns: type: CoreDNS etcd: local: dataDir: /var/lib/etcd imageRepository: k8s.gcr.io kind: ClusterConfiguration kubernetesVersion: v1.18.0 imageRepository: registry.cn-hangzhou.aliyuncs.com/google_containers controlPlaneEndpoint: 10.103.22.236:8443 apiServer: certSANs: - 10.10.18.45 - 10.10.18.46 - 10.10.16.249 - 127.0.0.1 etcd: external: endpoints: - https://10.10.18.42:2379 - https://10.10.18.43:2379 - https://10.10.18.44:2379 caFile: /etc/kubernetes/pki/etcd_bak/ca.crt certFile: /etc/kubernetes/pki/etcd_bak/server.crt keyFile: /etc/kubernetes/pki/etcd_bak/server.key networking: dnsDomain: cluster.local serviceSubnet: 10.96.0.0/12 scheduler: {}
[root@host-10-10-18-46 ~]# curl --cacert /etc/kubernetes/pki/etcd_bak/ca.crt --cert /etc/kubernetes/pki/etcd/server.crt --key /etc/kubernetes/pki/etcd/server.key -L https://10.10.18.44:2379/version curl: (35) Encountered end of file
[root@host-10-10-18-46 ~]# curl --cacert /etc/kubernetes/pki/etcd_bak/ca.crt --cert /etc/kubernetes/pki/etcd/server.crt --key /etc/kubernetes/pki/etcd/server.key -L https://10.10.18.44:2379/version -v * About to connect() to 10.10.18.44 port 2379 (#0) * Trying 10.10.18.44... * Connected to 10.10.18.44 (10.10.18.44) port 2379 (#0) * Initializing NSS with certpath: sql:/etc/pki/nssdb * CAfile: /etc/kubernetes/pki/etcd_bak/ca.crt CApath: none * NSS error -5961 (PR_CONNECT_RESET_ERROR) * TCP connection reset by peer * Closing connection 0 curl: (35) TCP connection reset by peer
http没问题
[root@host-10-10-18-46 ~]# curl --cacert /etc/kubernetes/pki/etcd_bak/ca.crt --cert /etc/kubernetes/pki/etcd/peer.crt --key /etc/kubernetes/pki/etcd/peer.key -L http://10.10.18.44:2379/version {"etcdserver":"3.3.11","etcdcluster":"3.3.0"}
[root@host-10-10-18-46 ~]#
curl http://10.10.18.44:2379/version {"etcdserver":"3.3.11","etcdcluster":"3.3.0"}
[root@host-10-10-18-46 ~]#
root@ubuntu:~/etcd-v3.5.0-linux-arm64# ETCDCTL_API=3 ./etcdctl --cacert=/etc/kubernetes/pki/etcd/ca.crt --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key --endpoints=http://10.10.18.42:2379 member list 1829ea2c82ecd13e, started, etcd1, http://10.10.18.42:2380, http://10.10.18.42:2379, false 19ddebfcb3e299fd, started, etcd2, http://10.10.18.43:2380, http://10.10.18.43:2379, false fe3b541533812c5d, started, etcd3, http://10.10.18.44:2380, http://10.10.18.44:2379, false root@ubuntu:~/etcd-v3.5.0-linux-arm64# ETCDCTL_API=3 ./etcdctl --cacert=/etc/kubernetes/pki/etcd/ca.crt --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key --endpoints=http://10.10.18.43:2379 member list 1829ea2c82ecd13e, started, etcd1, http://10.10.18.42:2380, http://10.10.18.42:2379, false 19ddebfcb3e299fd, started, etcd2, http://10.10.18.43:2380, http://10.10.18.43:2379, false fe3b541533812c5d, started, etcd3, http://10.10.18.44:2380, http://10.10.18.44:2379, false root@ubuntu:~/etcd-v3.5.0-linux-arm64# ETCDCTL_API=3 ./etcdctl --cacert=/etc/kubernetes/pki/etcd/ca.crt --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key --endpoints=http://10.10.18.44:2379 member list 1829ea2c82ecd13e, started, etcd1, http://10.10.18.42:2380, http://10.10.18.42:2379, false 19ddebfcb3e299fd, started, etcd2, http://10.10.18.43:2380, http://10.10.18.43:2379, false fe3b541533812c5d, started, etcd3, http://10.10.18.44:2380, http://10.10.18.44:2379, false root@ubuntu:~/etcd-v3.5.0-linux-arm64#
可以访问10.10.16.82:2379
[root@host-10-10-18-42 system]# curl --cacert /etc/kubernetes/pki/etcd/ca.crt --cert /etc/kubernetes/pki/etcd/server.crt --key /etc/kubernetes/pki/etcd/server.key https://10.10.18.42:2379/version curl: (35) Encountered end of file [root@host-10-10-18-42 system]# curl --cacert /etc/kubernetes/pki/etcd/ca.crt --cert /etc/kubernetes/pki/etcd/server.crt --key /etc/kubernetes/pki/etcd/server.key https://10.10.16.82:2379/version {"etcdserver":"3.4.3","etcdcluster":"3.4.0"}
[root@host-10-10-18-42 system]#
root@ubuntu:/etc# ps -elf | grep etcd | grep client-cert-auth 4 S root 7969 7939 2 80 0 - 2672731 futex_ Jun18 ? 05:44:28 etcd --advertise-client-urls=https://10.10.16.82:2379 --cert-file=/etc/kubernetes/pki/etcd/server.crt --client-cert-auth=true --data-dir=/var/lib/etcd --initial-advertise-peer-urls=https://10.10.16.82:2380 --initial-cluster=ubuntu=https://10.10.16.82:2380 --key-file=/etc/kubernetes/pki/etcd/server.key --listen-client-urls=https://127.0.0.1:2379,https://10.10.16.82:2379 --listen-metrics-urls=http://127.0.0.1:2381 --listen-peer-urls=https://10.10.16.82:2380 --name=ubuntu --peer-cert-file=/etc/kubernetes/pki/etcd/peer.crt --peer-client-cert-auth=true --peer-key-file=/etc/kubernetes/pki/etcd/peer.key --peer-trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt --snapshot-count=10000 --trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt
原来是没有开放https
重新生成etcd证书
创建基于根证书的config配置文件
#ca办法证书机构
cat > ca-config.json <<EOF
{
"signing": { "default": { "expiry": "87600h" }, "profiles": { "www": { "expiry": "87600h", "usages": [ "signing", "key encipherment", "server auth", "client auth" ] } }
}
}
EOF
#ca机构请求
cat > ca-csr.json <<EOF
{ "CN": "etcd CA", "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "L": "Beijing", "ST": "Beijing" } ]
}
EOF
#生成证书：读取上边两个文件生成证书
cfssl gencert -initca ca-csr.json | cfssljson -bare ca -root@ubuntu:~/cfssl/etcd# cat > ca-config.json <<EOF > { > "signing": { "default": { "expiry": "87600h" #证书过期时间h单位 }, "profiles": { "www": { "expiry": "87600h", "usages": [ "signing", "key encipherment", "server auth", "client auth" ] } } > } > } > EOF root@ubuntu:~/cfssl/etcd# cat > ca-csr.json <<EOF > { "CN": "etcd CA", "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "L": "Beijing", "ST": "Beijing" } ] > } > EOF root@ubuntu:~/cfssl/etcd# ./cfssl gencert -initca ca-csr.json | ./cfssljson -bare ca - 2021/06/30 10:43:36 [INFO] generating a new CA key and certificate from CSR 2021/06/30 10:43:36 [INFO] generate received request 2021/06/30 10:43:36 [INFO] received CSR 2021/06/30 10:43:36 [INFO] generating key: rsa-2048 2021/06/30 10:43:37 [INFO] encoded CSR 2021/06/30 10:43:37 [INFO] signed certificate with serial number 53627328402430641884101375169327098053785759268 root@ubuntu:~/cfssl/etcd#
创建生成etcd自签证书peer的csr的json配置文件
#etcd域名证书，需要把etcd节点ip都写进去，多写点备份用cat > server-csr.json <<EOF { "CN": "etcd", "hosts": [ "10.10.18.42", "10.10.18.43", "10.10.18.44" ], "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "L": "BeiJing", "ST": "BeiJing" } ] } EOF
root@ubuntu:~/cfssl/etcd# ./cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=www server-csr.json | ./cfssljson -bare server 2021/06/30 10:46:48 [INFO] generate received request 2021/06/30 10:46:48 [INFO] received CSR 2021/06/30 10:46:48 [INFO] generating key: rsa-2048 2021/06/30 10:46:49 [INFO] encoded CSR 2021/06/30 10:46:49 [INFO] signed certificate with serial number 277831989248432604565440323258702823212559696597
cat <<EOF >/usr/lib/systemd/system/etcd.service [Unit] Description=Etcd Server After=network.target After=network-online.target Wants=network-online.target [Service] Type=notify EnvironmentFile=/etc/etcd/etcd.conf ExecStart=/usr/bin/etcd --name=etcd01 --data-dir=${ETCD_DATA_DIR} --listen-peer-urls=${ETCD_LISTEN_PEER_URLS} --listen-client-urls=${ETCD_LISTEN_CLIENT_URLS},http://127.0.0.1:2379 --advertise-client-urls=${ETCD_ADVERTISE_CLIENT_URLS} --initial-advertise-peer-urls=${ETCD_INITIAL_ADVERTISE_PEER_URLS} --initial-cluster=${ETCD_INITIAL_CLUSTER} --initial-cluster-token=${ETCD_INITIAL_CLUSTER_TOKEN} --initial-cluster-state=new --cert-file=/opt/etcd/ssl/server.pem --key-file=/opt/etcd/ssl/server-key.pem --peer-cert-file=/opt/etcd/ssl/server.pem --peer-key-file=/opt/etcd/ssl/server-key.pem --trusted-ca-file=/opt/etcd/ssl/ca.pem --peer-trusted-ca-file=/opt/etcd/ssl/ca.pem Restart=on-failure LimitNOFILE=65536 [Install] WantedBy=multi-user.target EOF
[root@host-10-10-18-43 ~]# systemctl restart etcd Job for etcd.service failed because a timeout was exceeded. See "systemctl status etcd.service" and "journalctl -xe" for details. [root@host-10-10-18-43 ~]# journalctl -xe Jun 30 11:05:58 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.42:57514" (error "tls: first record does not look like a TLS handshake", ServerName "") Jun 30 11:05:58 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.44:44110" (error "tls: first record does not look like a TLS handshake", ServerName "") Jun 30 11:05:58 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.44:44112" (error "tls: first record does not look like a TLS handshake", ServerName "") Jun 30 11:05:58 host-10-10-18-43 etcd[20044]: 19ddebfcb3e299fd is starting a new election at term 33312 Jun 30 11:05:58 host-10-10-18-43 etcd[20044]: 19ddebfcb3e299fd became candidate at term 33313 Jun 30 11:05:58 host-10-10-18-43 etcd[20044]: 19ddebfcb3e299fd received MsgVoteResp from 19ddebfcb3e299fd at term 33313 Jun 30 11:05:58 host-10-10-18-43 etcd[20044]: 19ddebfcb3e299fd [logterm: 275, index: 25] sent MsgVote request to 1829ea2c82ecd13e at term 33313 Jun 30 11:05:58 host-10-10-18-43 etcd[20044]: 19ddebfcb3e299fd [logterm: 275, index: 25] sent MsgVote request to fe3b541533812c5d at term 33313 Jun 30 11:05:58 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.42:57526" (error "tls: first record does not look like a TLS handshake", ServerName "") Jun 30 11:05:58 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.44:44120" (error "tls: first record does not look like a TLS handshake", ServerName "") Jun 30 11:05:58 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.44:44122" (error "tls: first record does not look like a TLS handshake", ServerName "") Jun 30 11:05:58 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.42:57524" (error "tls: first record does not look like a TLS handshake", ServerName "") Jun 30 11:05:58 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.42:57536" (error "tls: first record does not look like a TLS handshake", ServerName "") Jun 30 11:05:58 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.42:57538" (error "tls: first record does not look like a TLS handshake", ServerName "") Jun 30 11:05:58 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.44:44130" (error "tls: first record does not look like a TLS handshake", ServerName "") Jun 30 11:05:58 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.44:44128" (error "tls: first record does not look like a TLS handshake", ServerName "") Jun 30 11:05:59 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.44:44140" (error "tls: first record does not look like a TLS handshake", ServerName "") Jun 30 11:05:59 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.42:57548" (error "tls: first record does not look like a TLS handshake", ServerName "") Jun 30 11:05:59 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.44:44138" (error "tls: first record does not look like a TLS handshake", ServerName "") Jun 30 11:05:59 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.42:57546" (error "tls: first record does not look like a TLS handshake", ServerName "") Jun 30 11:05:59 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.42:57558" (error "tls: first record does not look like a TLS handshake", ServerName "") Jun 30 11:05:59 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.44:44148" (error "tls: first record does not look like a TLS handshake", ServerName "") Jun 30 11:05:59 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.44:44150" (error "tls: first record does not look like a TLS handshake", ServerName "") Jun 30 11:05:59 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.42:57556" (error "tls: first record does not look like a TLS handshake", ServerName "") Jun 30 11:05:59 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.44:44158" (error "tls: first record does not look like a TLS handshake", ServerName "") Jun 30 11:05:59 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.44:44156" (error "tls: first record does not look like a TLS handshake", ServerName "") Jun 30 11:05:59 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.42:57566" (error "tls: first record does not look like a TLS handshake", ServerName "") Jun 30 11:05:59 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.42:57568" (error "tls: first record does not look like a TLS handshake", ServerName "") Jun 30 11:05:59 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.44:44166" (error "tls: first record does not look like a TLS handshake", ServerName "") Jun 30 11:05:59 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.44:44164" (error "tls: first record does not look like a TLS handshake", ServerName "") Jun 30 11:05:59 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.42:57580" (error "tls: first record does not look like a TLS handshake", ServerName "") Jun 30 11:05:59 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.42:57578" (error "tls: first record does not look like a TLS handshake", ServerName "") Jun 30 11:06:00 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.42:57620" (error "tls: oversized record received with length 21536", ServerName "") Jun 30 11:06:00 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.44:44172" (error "tls: first record does not look like a TLS handshake", ServerName "") Jun 30 11:06:00 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.44:44174" (error "tls: first record does not look like a TLS handshake", ServerName "") Jun 30 11:06:00 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.42:57590" (error "tls: first record does not look like a TLS handshake", ServerName "") Jun 30 11:06:00 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.42:57592" (error "tls: first record does not look like a TLS handshake", ServerName "") Jun 30 11:06:00 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.42:57646" (error "tls: first record does not look like a TLS handshake", ServerName "") Jun 30 11:06:00 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.44:44208" (error "tls: first record does not look like a TLS handshake", ServerName "") Jun 30 11:06:00 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.44:44210" (error "tls: first record does not look like a TLS handshake", ServerName "") Jun 30 11:06:00 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.42:57644" (error "tls: first record does not look like a TLS handshake", ServerName "") Jun 30 11:06:00 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.42:57658" (error "tls: first record does not look like a TLS handshake", ServerName "") Jun 30 11:06:00 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.44:44218" (error "tls: first record does not look like a TLS handshake", ServerName "") Jun 30 11:06:00 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.44:44220" (error "tls: first record does not look like a TLS handshake", ServerName "") Jun 30 11:06:00 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.42:57656" (error "tls: first record does not look like a TLS handshake", ServerName "") Jun 30 11:06:00 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.42:57666" (error "tls: first record does not look like a TLS handshake", ServerName "")
删除旧的
[root@host-10-10-18-43 ~]# rm /data/k8s/etcd/wal/* -rf [root@host-10-10-18-43 ~]# rm /data/k8s/etcd/data/* -rf
[root@host-10-10-18-43 ~]# systemctl status etcd ● etcd.service - Etcd Server Loaded: loaded (/usr/lib/systemd/system/etcd.service; disabled; vendor preset: disabled) Active: active (running) since Wed 2021-06-30 11:14:58 CST; 1min 13s ago Main PID: 20226 (etcd) CGroup: /system.slice/etcd.service └─20226 /usr/bin/etcd --name=etcd2 --data-dir=/data/k8s/etcd/data
访问成功
[root@host-10-10-18-46 ~]# curl --cacert /opt/etcd/ssl/ca.pem --cert /opt/etcd/ssl/server.pem --key /opt/etcd/ssl/server-key.pem https://10.10.18.42:2379/version {"etcdserver":"3.3.11","etcdcluster":"3.3.0"}[root@host-10-10-18-46 ~]#
kubeadm init --config
init失败查看kubelet
Jun 30 11:32:14 host-10-10-18-46 kubelet[25210]: E0630 11:32:14.112133 25210 kubelet.go:2267] node "host-10-10-18-46" not found Jun 30 11:32:14 host-10-10-18-46 kubelet[25210]: E0630 11:32:14.212346 25210 kubelet.go:2267] node "host-10-10-18-46" not found Jun 30 11:32:14 host-10-10-18-46 kubelet[25210]: E0630 11:32:14.312579 25210 kubelet.go:2267] node "host-10-10-18-46" not found Jun 30 11:32:14 host-10-10-18-46 kubelet[25210]: E0630 11:32:14.412767 25210 kubelet.go:2267] node "host-10-10-18-46" not found Jun 30 11:32:14 host-10-10-18-46 kubelet[25210]: E0630 11:32:14.512983 25210 kubelet.go:2267] node "host-10-10-18-46" not found Jun 30 11:32:14 host-10-10-18-46 kubelet[25210]: E0630 11:32:14.613160 25210 kubelet.go:2267] node "host-10-10-18-46" not found Jun 30 11:32:14 host-10-10-18-46 kubelet[25210]: E0630 11:32:14.713375 25210 kubelet.go:2267] node "host-10-10-18-46" not found Jun 30 11:32:14 host-10-10-18-46 kubelet[25210]: E0630 11:32:14.813574 25210 kubelet.go:2267] node "host-10-10-18-46" not found Jun 30 11:32:14 host-10-10-18-46 kubelet[25210]: E0630 11:32:14.913774 25210 kubelet.go:2267] node "host-10-10-18-46" not found Jun 30 11:32:15 host-10-10-18-46 kubelet[25210]: E0630 11:32:15.013968 25210 kubelet.go:2267] node "host-10-10-18-46" not found Jun 30 11:32:15 host-10-10-18-46 kubelet[25210]: E0630 11:32:15.114144 25210 kubelet.go:2267] node "host-10-10-18-46" not found Jun 30 11:32:15 host-10-10-18-46 kubelet[25210]: E0630 11:32:15.214331 25210 kubelet.go:2267] node "host-10-10-18-46" not found Jun 30 11:32:15 host-10-10-18-46 kubelet[25210]: E0630 11:32:15.314539 25210 kubelet.go:2267] node "host-10-10-18-46" not found Jun 30 11:32:15 host-10-10-18-46 kubelet[25210]: E0630 11:32:15.414737 25210 kubelet.go:2267] node "host-10-10-18-46" not found Jun 30 11:32:15 host-10-10-18-46 kubelet[25210]: E0630 11:32:15.514889 25210 kubelet.go:2267] node "host-10-10-18-46" not found Jun 30 11:32:15 host-10-10-18-46 kubelet[25210]: E0630 11:32:15.615078 25210 kubelet.go:2267] node "host-10-10-18-46" not found Jun 30 11:32:15 host-10-10-18-46 kubelet[25210]: E0630 11:32:15.715240 25210 kubelet.go:2267] node "host-10-10-18-46" not found
[root@host-10-10-18-46 ~]# cat /etc/kubernetes/kubelet.conf apiVersion: v1 clusters: - cluster: certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUN5RENDQWJDZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKY201bGRHVnpNQjRYRFRJeE1EWXpNREF6TWpNek5Gb1hEVE14TURZeU9EQXpNak16TkZvd0ZURVRNQkVHQTFVRQpBeE1LYTNWaVpYSnVaWFJsY3pDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTUt0CndHbXB2bTBPaFFPMjVBeFpJM295Z2FRS0VGSk81c0JSMmVoem1yN2ZVNlBaWWhrb1BoTXZEZ3RCNmdnNlBjQkcKNFB3M2JlQnNBZXZGaThkNEJ0bFVLeTdJVTFrZHdtcldMTHZBT3lRVnJveExSQ0V0QUVMNWlyUENYQmFjZVBTbwpRV3lnRUFYTEQvTkNOb0NndDF1a3RYSEVHNTlWdG1RbmtiSitnVGNpK1FrWnl5MGRQOWUyOE83SjRIcUhUNHo5CkVRNTlUamprdWVid2VaUmF6WVFYQTV1TWZHY2tJK05VQytCank0NHhQYnNTL2FRSnJPM1c2NzQydTJtdXFXblEKUmZBRHJLOGhMODRVWW4vL1ZReWM4cjFNWENEVXRBd0gyU3dROE1EZTJFM3VURGNyU29HSWx4RXJvelR3Y3ZCNgoweDQwVXAwSEhXZ0NQOVF4Ulk4Q0F3RUFBYU1qTUNFd0RnWURWUjBQQVFIL0JBUURBZ0trTUE4R0ExVWRFd0VCCi93UUZNQU1CQWY4d0RRWUpLb1pJaHZjTkFRRUxCUUFEZ2dFQkFKQS9NVVhVVVU0K0ZmVFExaHROQ2JHeUF2SjMKZjZLOHAzUHZEdnQvR2xwWXFMZkRhaFltWHh3ZEsyaFNVMGliRFZkMW1vem0xL3dmenYzbTl2Z1dKR09rTFhVOQpoSlZkNWVGM0IyNWRkWGdhOGVvVVFJdWNMV2t3QklrSmtITnpiRUJ5UmVlTEV4WUZKN205bXFKa1Z4SVN6Rm1FClN6MG96cXRMQUtHaWZSTnhUbXQvbjQ3RjJma2psNmlYRDlpOGx5WmNyOUxVZklIcTVldFYvYmNRbWdOQ01yZXcKeGZ5R3h1YVgxZ2NQT2JITmVQMUUxcXljOHI5dWU3RWFzSFlhaTY4REFTVWxFalJJMXRxUDgwYkZRSHQzRU5xaAp0ckFKdzIzVzhEcTRibHlDdld1YTBCanB0SG1pWFY0UENVZ1dvK3VsUEVWWXVibzlXbC9jUnZxOENMWT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= server: https://10.103.22.236:8443 name: kubernetes
[root@host-10-10-18-46 ~]# cat kubeadm-init.yaml.yaml apiVersion: kubeadm.k8s.io/v1beta2 bootstrapTokens: - groups: - system:bootstrappers:kubeadm:default-node-token token: abcdef.0123456789abcdef ttl: 24h0m0s usages: - signing - authentication kind: InitConfiguration localAPIEndpoint: advertiseAddress: 1.2.3.4 bindPort: 6443 nodeRegistration: criSocket: /var/run/dockershim.sock name: host-10-10-18-46 taints: - effect: NoSchedule key: node-role.kubernetes.io/master --- apiServer: timeoutForControlPlane: 4m0s apiVersion: kubeadm.k8s.io/v1beta2 certificatesDir: /etc/kubernetes/pki clusterName: kubernetes controllerManager: {} dns: type: CoreDNS etcd: local: dataDir: /var/lib/etcd imageRepository: k8s.gcr.io kind: ClusterConfiguration kubernetesVersion: v1.18.0 imageRepository: registry.cn-hangzhou.aliyuncs.com/google_containers controlPlaneEndpoint: 10.103.22.236:8443
[root@host-10-10-18-46 ~]# netstat -pan | grep 6443 tcp 0 1 10.10.18.46:45042 1.2.3.4:6443 SYN_SENT 25210/kubelet tcp 0 1 10.10.18.46:45024 1.2.3.4:6443 SYN_SENT 25210/kubelet tcp6 0 0 :::6443 :::* LISTEN 27229/kube-apiserve tcp6 0 0 ::1:6443 ::1:55698 ESTABLISHED 27229/kube-apiserve tcp6 0 0 ::1:55698 ::1:6443 ESTABLISHED 27229/kube-apiserve unix 3 [ ] STREAM CONNECTED 36443 1/systemd /run/systemd/journal/stdout [root@host-10-10-18-46 ~]#
另外一个节点
[root@host-10-10-18-45 ~]# kubeadm version kubeadm version: &version.Info{Major:"1", Minor:"18", GitVersion:"v1.18.1", GitCommit:"7879fc12a63337efff607952a323df90cdc7a335", GitTreeState:"clean", BuildDate:"2020-04-08T17:36:32Z", GoVersion:"go1.13.9", Compiler:"gc", Platform:"linux/arm64"} [root@host-10-10-18-45 ~]# netstat -pan | grep 6443 [root@host-10-10-18-45 ~]#
配置keepalived + haproxy
#--------------------------------------------------------------------- # Example configuration for a possible web application. See the # full configuration options online. # # https://www.haproxy.org/download/2.1/doc/configuration.txt # https://cbonte.github.io/haproxy-dconv/2.1/configuration.html# #--------------------------------------------------------------------- #--------------------------------------------------------------------- # Global settings #--------------------------------------------------------------------- global # to have these messages end up in /var/log/haproxy.log you will # need to: # # 1) configure syslog to accept network log events. This is done # by adding the '-r' option to the SYSLOGD_OPTIONS in # /etc/sysconfig/syslog # # 2) configure local2 events to go to the /var/log/haproxy.log # file. A line like the following can be added to # /etc/sysconfig/syslog # # local2.* /var/log/haproxy.log # log 127.0.0.1 local2 # chroot /var/lib/haproxy pidfile /var/run/haproxy.pid maxconn 4000 # user haproxy # group haproxy # daemon # turn on stats unix socket stats socket /var/lib/haproxy/stats #--------------------------------------------------------------------- # common defaults that all the 'listen' and 'backend' sections will # use if not designated in their block #--------------------------------------------------------------------- defaults mode http log global option httplog option dontlognull option http-server-close option forwardfor except 127.0.0.0/8 option redispatch retries 3 timeout http-request 10s timeout queue 1m timeout connect 10s timeout client 1m timeout server 1m timeout http-keep-alive 10s timeout check 10s maxconn 3000 #--------------------------------------------------------------------- # main frontend which proxys to the backends #--------------------------------------------------------------------- frontend frr mode tcp bind *:9443 ## 监听9443端口 # bind *:443 ssl # To be completed .... acl url_static path_beg -i /static /images /javascript /stylesheets acl url_static path_end -i .jpg .gif .png .css .js default_backend kube-apiserver #--------------------------------------------------------------------- # round robin balancing between the various backends #--------------------------------------------------------------------- backend kube-apiserver mode tcp option tcplog option tcp-check balance roundrobin default-server inter 10s downinter 5s rise 2 fall 2 slowstart 60s maxconn 250 maxqueue 256 weight 100 server host-10-10-18-46 10.10.18.46:6443 check
暂时只配置一个server
keepavlivede vip
root@ubuntu:/etc/haproxy# cat ../keepalived/keepalived.conf global_defs { script_user root enable_script_security } vrrp_script chk_haproxy { script "/bin/bash -c 'if [[ $(netstat -nlp | grep 9443 | wc -l) ]]; then exit 0; else exit 1; fi'" # haproxy 检测 interval 2 # 每2秒执行一次检测 #weight -10 # 权重变化 } vrrp_instance VI_1 { interface enahisic2i0 ###宿主机网卡名 state BACKUP virtual_router_id 61 # id设为相同，表示是同一个虚拟路由组 priority 80 #初始权重 nopreempt #不抢占 unicast_peer { 10.10.16.47 10.10.16.251 } virtual_ipaddress { 10.10.16.249 # vip } authentication { auth_type PASS auth_pass password } track_script { chk_haproxy } #notify "/container/service/keepalived/assets/" }
访问keepalived vip
root@ubuntu:/etc/haproxy# telnet 10.10.16.249 9443 Trying 10.10.16.249... Connected to 10.10.16.249. Escape character is '^]'. ^C^CConnection closed by foreign host.
[root@host-10-10-18-46 ~]# kubeadm init --config kubeadm-init.yaml.yaml W0630 12:02:37.304175 1295 strict.go:54] error unmarshaling configuration schema.GroupVersionKind{Group:"kubeadm.k8s.io", Version:"v1beta2", Kind:"ClusterConfiguration"}: error converting YAML to JSON: yaml: unmarshal errors: line 15: key "imageRepository" already set in map line 18: key "apiServer" already set in map line 24: key "etcd" already set in map
Your Kubernetes control-plane has initialized successfully! To start using your cluster, you need to run the following as a regular user: mkdir -p $HOME/.kube sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config sudo chown $(id -u):$(id -g) $HOME/.kube/config You should now deploy a pod network to the cluster. Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at: https://kubernetes.io/docs/concepts/cluster-administration/addons/ You can now join any number of control-plane nodes by copying certificate authorities and service account keys on each node and then running the following as root: kubeadm join 10.10.16.249:9443 --token abcdef.0123456789abcdef --discovery-token-ca-cert-hash sha256:9bc0bcddb2b97791717943b714ffa410cb5963061889086f04 --control-plane Then you can join any number of worker nodes by running the following on each as root: kubeadm join 10.10.16.249:9443 --token abcdef.0123456789abcdef --discovery-token-ca-cert-hash sha256:9bc0bcddb2b97791717943b714ffa410cb5963061889
kubeadm-init.yaml
[root@host-10-10-18-46 ~]# cat kubeadm-init.yaml.yaml apiVersion: kubeadm.k8s.io/v1beta2 bootstrapTokens: - groups: - system:bootstrappers:kubeadm:default-node-token token: abcdef.0123456789abcdef ttl: 24h0m0s usages: - signing - authentication kind: InitConfiguration localAPIEndpoint: advertiseAddress: 1.2.3.4 --默认 bindPort: 6443 nodeRegistration: criSocket: /var/run/dockershim.sock name: host-10-10-18-46 taints: - effect: NoSchedule key: node-role.kubernetes.io/master --- apiServer: timeoutForControlPlane: 4m0s apiVersion: kubeadm.k8s.io/v1beta2 certificatesDir: /etc/kubernetes/pki clusterName: kubernetes controllerManager: {} dns: type: CoreDNS etcd: local: dataDir: /var/lib/etcd imageRepository: k8s.gcr.io kind: ClusterConfiguration kubernetesVersion: v1.18.0 imageRepository: registry.cn-hangzhou.aliyuncs.com/google_containers controlPlaneEndpoint: 10.10.16.249:9443 apiServer: certSANs: - 10.10.18.45 - 10.10.18.46 - 10.10.16.249 - 127.0.0.1 etcd: external: endpoints: - https://10.10.18.42:2379 - https://10.10.18.43:2379 - https://10.10.18.44:2379 caFile: /opt/etcd/ssl/ca.pem certFile: /opt/etcd/ssl/server.pem keyFile: /opt/etcd/ssl/server-key.pem networking: dnsDomain: cluster.local serviceSubnet: 10.96.0.0/12 scheduler: {}
[root@host-10-10-18-46 ~]# ps -elf | grep apiserver 0 S root 31133 21547 0 80 0 - 1724 pipe_w 14:18 pts/0 00:00:00 grep --color=auto apiserver [root@host-10-10-18-46 ~]# kubectl -n kube-system get pods -o wide The connection to the server 10.10.18.46:6443 was refused - did you specify the right host or port? [root@host-10-10-18-46 ~]# netstat -pan | grep 6443 unix 3 [ ] STREAM CONNECTED 36443 1/systemd /run/systemd/journal/stdout [root@host-10-10-18-46 ~]# ps -elf | grep apiserver 0 S root 31196 21547 0 80 0 - 1724 pipe_w 14:18 pts/0 00:00:00 grep --color=auto apiserver [root@host-10-10-18-46 ~]#
重启kubelet
systemctl restart kubelet
[root@host-10-10-18-46 ~]# ps -elf | grep apiserver 4 S root 31884 31863 29 80 0 - 7681 futex_ 14:19 ? 00:00:13 kube-apiserver --advertise-address=1.2.3.4 --allow-privileged=true --authorization-mode=Node,RBAC --client-ca-file=/etc/kubernetes/pki/ca.crt --enable-admission-plugins=NodeRestriction --enable-bootstrap-token-auth=true --etcd-cafile=/opt/etcd/ssl/ca.pem --etcd-certfile=/opt/etcd/ssl/server.pem --etcd-keyfile=/opt/etcd/ssl/server-key.pem --etcd-servers=https://10.10.18.42:2379,https://10.10.18.43:2379,https://10.10.18.44:2379 --insecure-port=0 --kubelet-client-certificate=/etc/kubernetes/pki/apiserver-kubelet-client.crt --kubelet-client-key=/etc/kubernetes/pki/apiserver-kubelet-client.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.crt --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client.key --requestheader-allowed-names=front-proxy-client --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-key-file=/etc/kubernetes/pki/sa.pub --service-cluster-ip-range=10.96.0.0/12 --tls-cert-file=/etc/kubernetes/pki/apiserver.crt --tls-private-key-file=/etc/kubernetes/pki/apiserver.key 0 S root 32191 21547 0 80 0 - 1724 pipe_w 14:20 pts/0 00:00:00 grep --color=auto apiserver [root@host-10-10-18-46 ~]# netstat -pan | grep 6443 tcp 0 1 10.10.18.46:48926 1.2.3.4:6443 SYN_SENT 31315/kubelet tcp 0 1 10.10.18.46:48936 1.2.3.4:6443 SYN_SENT 31315/kubelet tcp6 0 0 :::6443 :::* LISTEN 31884/kube-apiserve tcp6 0 0 10.10.18.46:6443 10.10.16.82:42914 ESTABLISHED 31884/kube-apiserve tcp6 0 0 ::1:6443 ::1:59596 ESTABLISHED 31884/kube-apiserve tcp6 0 0 ::1:59596 ::1:6443 ESTABLISHED 31884/kube-apiserve tcp6 0 0 10.10.18.46:6443 10.10.16.82:42906 ESTABLISHED 31884/kube-apiserve tcp6 0 0 10.10.18.46:6443 10.10.16.82:42930 ESTABLISHED 31884/kube-apiserve tcp6 0 0 10.10.18.46:6443 10.10.16.82:42966 ESTABLISHED 31884/kube-apiserve tcp6 0 0 10.10.18.46:6443 10.10.16.82:42900 ESTABLISHED 31884/kube-apiserve
配置执行 kubectl 命令用户
[root@host-10-10-18-46 ~]# kubectl -n kube-system get pods -o wide Unable to connect to the server: x509: certificate signed by unknown authority (possibly because of "crypto/rsa: verification error" while trying to verify candidate authority certificate "kubernetes") [root@host-10-10-18-46 ~]# mkdir -p $HOME/.kube [root@host-10-10-18-46 ~]# cp -i /etc/kubernetes/admin.conf $HOME/.kube/config cp: overwrite ‘/root/.kube/config’? y [root@host-10-10-18-46 ~]# chown $(id -u):$(id -g) $HOME/.kube/config [root@host-10-10-18-46 ~]# kubectl -n kube-system get pods -o wide NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES coredns-546565776c-ch9n7 0/1 Pending 0 135m <none> <none> <none> <none> coredns-546565776c-dddl9 0/1 Pending 0 135m <none> <none> <none> <none> kube-apiserver-host-10-10-18-46 1/1 Running 34 139m 10.10.18.46 host-10-10-18-46 <none> <none> kube-controller-manager-host-10-10-18-46 1/1 Running 25 139m 10.10.18.46 host-10-10-18-46 <none> <none> kube-proxy-zl8fw 1/1 Running 0 135m 10.10.18.46 host-10-10-18-46 <none> <none> kube-scheduler-host-10-10-18-46 1/1 Running 25 139m 10.10.18.46 host-10-10-18-46 <none> <none> [root@host-10-10-18-46 ~]#
查看集群状态
[root@host-10-10-18-46 ~]# kubectl get cs NAME STATUS MESSAGE ERROR scheduler Unhealthy Get http://127.0.0.1:10251/healthz: dial tcp 127.0.0.1:10251: connect: connection refused controller-manager Healthy ok etcd-1 Healthy {"health":"true"} etcd-2 Healthy {"health":"true"} etcd-0 Healthy {"health":"true"} [root@host-10-10-18-46 ~]#
root@ubuntu:~/cfssl/etcd# ETCDCTL_API=3 ./etcdctl --cacert ./etcd/ca.pem --cert ./etcd/server.pem --key ./etcd/server-key.pem --endpoints=https://10.10.18.42:2379 get / --prefix --keys-only | more /registry/apiregistration.k8s.io/apiservices/v1. /registry/apiregistration.k8s.io/apiservices/v1.admissionregistration.k8s.io /registry/apiregistration.k8s.io/apiservices/v1.apiextensions.k8s.io /registry/apiregistration.k8s.io/apiservices/v1.apps /registry/apiregistration.k8s.io/apiservices/v1.authentication.k8s.io /registry/apiregistration.k8s.io/apiservices/v1.authorization.k8s.io /registry/apiregistration.k8s.io/apiservices/v1.autoscaling /registry/apiregistration.k8s.io/apiservices/v1.batch
root@ubuntu:~/cfssl/etcd# ETCDCTL_API=3 ./etcdctl --cacert ./etcd/ca.pem --cert ./etcd/server.pem --key ./etcd/server-key.pem --endpoints=https://10.10.18.42:2379 get /registry/clusterrolebindings/kubeadm:get-nodes /registry/clusterrolebindings/kubeadm:get-nodes k8s 2 rbac.authorization.k8s.io/v1ClusterRoleBindingkubeadm:get-nodes"*$a0766228-3694-4906-9787-b2ca2b181b7b2z kubeadmUpdaterbac.authorization.k8s.io/vFieldsV1:I G{"f:roleRef":{"f:apiGroup":{},"f:kind":{},"f:name":{}},"f:subjects":{}}U Grouprbac.authorization.k8s.io/system:bootstrappers:kubeadm:default-node-token"; rbac.authorization.k8s.io ClusterRolekubeadm:get-nodes"
[root@host-10-10-18-46 ~]# kubectl -n kube-system get nodes NAME STATUS ROLES AGE VERSION host-10-10-18-46 NotReady master 157m v1.18.1 [root@host-10-10-18-46 ~]#
[root@host-10-10-18-46 ~]# kubectl -n kube-system get pods -o wide NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES coredns-546565776c-ch9n7 0/1 Pending 0 157m <none> <none> <none> <none> coredns-546565776c-dddl9 0/1 Pending 0 157m <none> <none> <none> <none> kube-apiserver-host-10-10-18-46 1/1 Running 41 160m 10.10.18.46 host-10-10-18-46 <none> <none> kube-controller-manager-host-10-10-18-46 1/1 Running 31 160m 10.10.18.46 host-10-10-18-46 <none> <none> kube-proxy-zl8fw 1/1 Running 0 157m 10.10.18.46 host-10-10-18-46 <none> <none> kube-scheduler-host-10-10-18-46 1/1 Running 31 160m 10.10.18.46 host-10-10-18-46 <none> <none> [root@host-10-10-18-46 ~]# kubectl -n kube-system describe coredns-546565776c-ch9n7 error: the server doesn't have a resource type "coredns-546565776c-ch9n7" [root@host-10-10-18-46 ~]# kubectl -n kube-system describe pods coredns-546565776c-ch9n7 Name: coredns-546565776c-ch9n7 Namespace: kube-system Priority: 2000000000 Priority Class Name: system-cluster-critical Node: <none> Labels: k8s-app=kube-dns pod-template-hash=546565776c Annotations: <none> Status: Pending IP: IPs: <none> Controlled By: ReplicaSet/coredns-546565776c Containers: coredns: Image: registry.cn-hangzhou.aliyuncs.com/google_containers/coredns:1.6.7 Ports: 53/UDP, 53/TCP, 9153/TCP Host Ports: 0/UDP, 0/TCP, 0/TCP Args: -conf /etc/coredns/Corefile Limits: memory: 170Mi Requests: cpu: 100m memory: 70Mi Liveness: http-get http://:8080/health delay=60s timeout=5s period=10s #success=1 #failure=5 Readiness: http-get http://:8181/ready delay=0s timeout=1s period=10s #success=1 #failure=3 Environment: <none> Mounts: /etc/coredns from config-volume (ro) /var/run/secrets/kubernetes.io/serviceaccount from coredns-token-gl9fl (ro) Conditions: Type Status PodScheduled False Volumes: config-volume: Type: ConfigMap (a volume populated by a ConfigMap) Name: coredns Optional: false coredns-token-gl9fl: Type: Secret (a volume populated by a Secret) SecretName: coredns-token-gl9fl Optional: false QoS Class: Burstable Node-Selectors: kubernetes.io/os=linux Tolerations: CriticalAddonsOnly node-role.kubernetes.io/master:NoSchedule node.kubernetes.io/not-ready:NoExecute for 300s node.kubernetes.io/unreachable:NoExecute for 300s Events: Type Reason Age From Message ---- ------ ---- ---- ------- Warning FailedScheduling <unknown> default-scheduler 0/1 nodes are available: 1 node(s) had taint {node.kubernetes.io/not-ready: }, that the pod didn't tolerate. Warning FailedScheduling <unknown> default-scheduler 0/1 nodes are available: 1 node(s) had taint {node.kubernetes.io/not-ready: }, that the pod didn't tolerate. Warning FailedScheduling <unknown> default-scheduler 0/1 nodes are available: 1 node(s) had taint {node.kubernetes.io/not-ready: }, that the pod didn't tolerate. Warning FailedScheduling <unknown> default-scheduler 0/1 nodes are available: 1 node(s) had taint {node.kubernetes.io/not-ready: }, that the pod didn't tolerate. Warning FailedScheduling <unknown> default-scheduler 0/1 nodes are available: 1 node(s) had taint {node.kubernetes.io/not-ready: }, that the pod didn't tolerate. Warning FailedScheduling <unknown> default-scheduler 0/1 nodes are available: 1 node(s) had taint {node.kubernetes.io/not-ready: }, that the pod didn't tolerate. Warning FailedScheduling <unknown> default-scheduler 0/1 nodes are available: 1 node(s) had taint {node.kubernetes.io/not-ready: }, that the pod didn't tolerate. Warning FailedScheduling <unknown> default-scheduler 0/1 nodes are available: 1 node(s) had taint {node.kubernetes.io/not-ready: }, that the pod didn't tolerate. Warning FailedScheduling <unknown> default-scheduler 0/1 nodes are available: 1 node(s) had taint {node.kubernetes.io/not-ready: }, that the pod didn't tolerate. Warning FailedScheduling <unknown> default-scheduler 0/1 nodes are available: 1 node(s) had taint {node.kubernetes.io/not-ready: }, that the pod didn't tolerate. Warning FailedScheduling <unknown> default-scheduler 0/1 nodes are available: 1 node(s) had taint {node.kubernetes.io/not-ready: }, that the pod didn't tolerate. Warning FailedScheduling <unknown> default-scheduler 0/1 nodes are available: 1 node(s) had taint {node.kubernetes.io/not-ready: }, that the pod didn't tolerate. Warning FailedScheduling <unknown> default-scheduler 0/1 nodes are available: 1 node(s) had taint {node.kubernetes.io/not-ready: }, that the pod didn't tolerate. Warning FailedScheduling <unknown> default-scheduler 0/1 nodes are available: 1 node(s) had taint {node.kubernetes.io/not-ready: }, that the pod didn't tolerate. Warning FailedScheduling <unknown> default-scheduler 0/1 nodes are available: 1 node(s) had taint {node.kubernetes.io/not-ready: }, that the pod didn't tolerate. Warning FailedScheduling <unknown> default-scheduler 0/1 nodes are available: 1 node(s) had taint {node.kubernetes.io/not-ready: }, that the pod didn't tolerate. Warning FailedScheduling <unknown> default-scheduler 0/1 nodes are available: 1 node(s) had taint {node.kubernetes.io/not-ready: }, that the pod didn't tolerate. Warning FailedScheduling <unknown> default-scheduler 0/1 nodes are available: 1 node(s) had taint {node.kubernetes.io/not-ready: }, that the pod didn't tolerate. Warning FailedScheduling <unknown> default-scheduler 0/1 nodes are available: 1 node(s) had taint {node.kubernetes.io/not-ready: }, that the pod didn't tolerate. Warning FailedScheduling <unknown> default-scheduler 0/1 nodes are available: 1 node(s) had taint {node.kubernetes.io/not-ready: }, that the pod didn't tolerate. Warning FailedScheduling 120m default-scheduler 0/1 nodes are available: 1 node(s) had taint {node.kubernetes.io/not-ready: }, that the pod didn't tolerate. Warning FailedScheduling 110m default-scheduler 0/1 nodes are available: 1 node(s) had taint {node.kubernetes.io/not-ready: }, that the pod didn't tolerate. Warning FailedScheduling 99m default-scheduler 0/1 nodes are available: 1 node(s) had taint {node.kubernetes.io/not-ready: }, that the pod didn't tolerate. Warning FailedScheduling 89m default-scheduler 0/1 nodes are available: 1 node(s) had taint {node.kubernetes.io/not-ready: }, that the pod didn't tolerate. Warning FailedScheduling 80m default-scheduler 0/1 nodes are available: 1 node(s) had taint {node.kubernetes.io/not-ready: }, that the pod didn't tolerate. Warning FailedScheduling 70m default-scheduler 0/1 nodes are available: 1 node(s) had taint {node.kubernetes.io/not-ready: }, that the pod didn't tolerate. Warning FailedScheduling 57m default-scheduler 0/1 nodes are available: 1 node(s) had taint {node.kubernetes.io/not-ready: }, that the pod didn't tolerate. Warning FailedScheduling 43m default-scheduler 0/1 nodes are available: 1 node(s) had taint {node.kubernetes.io/not-ready: }, that the pod didn't tolerate. Warning FailedScheduling 33m default-scheduler 0/1 nodes are available: 1 node(s) had taint {node.kubernetes.io/not-ready: }, that the pod didn't tolerate. Warning FailedScheduling 23m default-scheduler 0/1 nodes are available: 1 node(s) had taint {node.kubernetes.io/not-ready: }, that the pod didn't tolerate. Warning FailedScheduling 6m19s default-scheduler 0/1 nodes are available: 1 node(s) had taint {node.kubernetes.io/not-ready: }, that the pod didn't tolerate. [root@host-10-10-18-46 ~]#
允许master节点部署pod，使用命令如下:
[root@host-10-10-18-46 ~]# kubectl taint nodes --all node-role.kubernetes.io/master- node/host-10-10-18-46 untainted [root@host-10-10-18-46 ~]# kubectl -n kube-system get pods -o wide NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES coredns-546565776c-ch9n7 0/1 Pending 0 159m <none> <none> <none> <none> coredns-546565776c-dddl9 0/1 Pending 0 159m <none> <none> <none> <none> kube-apiserver-host-10-10-18-46 1/1 Running 42 162m 10.10.18.46 host-10-10-18-46 <none> <none> kube-controller-manager-host-10-10-18-46 1/1 Running 32 162m 10.10.18.46 host-10-10-18-46 <none> <none> kube-proxy-zl8fw 1/1 Running 0 159m 10.10.18.46 host-10-10-18-46 <none> <none> kube-scheduler-host-10-10-18-46 1/1 Running 32 162m 10.10.18.46 host-10-10-18-46 <none> <none>
[root@host-10-10-18-46 ~]# kubectl -n kube-system delete pods coredns-546565776c-ch9n7 coredns-546565776c-dddl9 pod "coredns-546565776c-ch9n7" deleted pod "coredns-546565776c-dddl9" deleted [root@host-10-10-18-46 ~]# kubectl -n kube-system get pods -o wide NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES coredns-546565776c-v49kt 0/1 Pending 0 3s <none> <none> <none> <none> coredns-546565776c-z5pq6 0/1 Pending 0 4s <none> <none> <none> <none> kube-apiserver-host-10-10-18-46 1/1 Running 42 163m 10.10.18.46 host-10-10-18-46 <none> <none> kube-controller-manager-host-10-10-18-46 1/1 Running 32 163m 10.10.18.46 host-10-10-18-46 <none> <none> kube-proxy-zl8fw 1/1 Running 0 160m 10.10.18.46 host-10-10-18-46 <none> <none> kube-scheduler-host-10-10-18-46 1/1 Running 32 163m 10.10.18.46 host-10-10-18-46 <none> <none>
没有paused
首先，我们看看需要安装哪些镜像，使用如下命令：
[root@host-10-10-18-46 ~]# kubeadm config images list I0630 15:03:59.166843 13472 version.go:252] remote version is much newer: v1.21.2; falling back to: stable-1.18 W0630 15:03:59.835027 13472 configset.go:202] WARNING: kubeadm cannot validate component configs for API groups [kubelet.config.k8s.io kubeproxy.config.k8s.io] k8s.gcr.io/kube-apiserver:v1.18.20 k8s.gcr.io/kube-controller-manager:v1.18.20 k8s.gcr.io/kube-scheduler:v1.18.20 k8s.gcr.io/kube-proxy:v1.18.20 k8s.gcr.io/pause:3.2 k8s.gcr.io/etcd:3.4.3-0 k8s.gcr.io/coredns:1.6.7
[root@host-10-10-18-46 ~]# kubectl -n kube-system get nodes NAME STATUS ROLES AGE VERSION host-10-10-18-46 NotReady master 166m v1.18.1 [root@host-10-10-18-46 ~]#
QoS Class: Burstable Node-Selectors: kubernetes.io/os=linux Tolerations: CriticalAddonsOnly node-role.kubernetes.io/master:NoSchedule node.kubernetes.io/not-ready:NoExecute for 300s node.kubernetes.io/unreachable:NoExecute for 300s Events: Type Reason Age From Message ---- ------ ---- ---- ------- Warning FailedScheduling <unknown> default-scheduler 0/1 nodes are available: 1 node(s) had taint {node.kubernetes.io/not-ready: }, that the pod didn't tolerate. Warning FailedScheduling <unknown> default-scheduler 0/1 nodes are available: 1 node(s) had taint {node.kubernetes.io/not-ready: }, that the pod didn't tolerate. [root@host-10-10-18-46 ~]#
[root@host-10-10-18-46 ~]# kubectl describe node host-10-10-18-46 Name: host-10-10-18-46 Roles: master Labels: beta.kubernetes.io/arch=arm64 beta.kubernetes.io/os=linux kubernetes.io/arch=arm64 kubernetes.io/hostname=host-10-10-18-46 kubernetes.io/os=linux node-role.kubernetes.io/master= Annotations: kubeadm.alpha.kubernetes.io/cri-socket: /var/run/dockershim.sock node.alpha.kubernetes.io/ttl: 0 volumes.kubernetes.io/controller-managed-attach-detach: true CreationTimestamp: Wed, 30 Jun 2021 12:03:35 +0800 Taints: node.kubernetes.io/not-ready:NoSchedule Unschedulable: false Lease: HolderIdentity: host-10-10-18-46 AcquireTime: <unset> RenewTime: Wed, 30 Jun 2021 14:54:05 +0800 Conditions: Type Status LastHeartbeatTime LastTransitionTime Reason Message ---- ------ ----------------- ------------------ ------ ------- MemoryPressure False Wed, 30 Jun 2021 14:49:22 +0800 Wed, 30 Jun 2021 12:03:35 +0800 KubeletHasSufficientMemory kubelet has sufficient memory available DiskPressure False Wed, 30 Jun 2021 14:49:22 +0800 Wed, 30 Jun 2021 12:03:35 +0800 KubeletHasNoDiskPressure kubelet has no disk pressure PIDPressure False Wed, 30 Jun 2021 14:49:22 +0800 Wed, 30 Jun 2021 12:03:35 +0800 KubeletHasSufficientPID kubelet has sufficient PID available Ready False Wed, 30 Jun 2021 14:49:22 +0800 Wed, 30 Jun 2021 12:03:35 +0800 KubeletNotReady runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized Addresses: InternalIP: 10.10.18.46 Hostname: host-10-10-18-46 Capacity: cpu: 4 ephemeral-storage: 7978Mi hugepages-2Mi: 0 hugepages-512Mi: 0 memory: 7756672Ki pods: 110 Allocatable: cpu: 4 ephemeral-storage: 7528985383 hugepages-2Mi: 0 hugepages-512Mi: 0 memory: 7654272Ki pods: 110 System Info: Machine ID: 30689d599b59462f9fee88051771bea5 System UUID: B80706BA-B199-4ED2-927B-66A6EC045417 Boot ID: 3205f1fc-6015-4fcd-a9c1-c9c24e2d8d80 Kernel Version: 4.14.0-115.el7a.0.1.aarch64 OS Image: CentOS Linux 7 (AltArch) Operating System: linux Architecture: arm64 Container Runtime Version: docker://20.10.7 Kubelet Version: v1.18.1 Kube-Proxy Version: v1.18.1 Non-terminated Pods: (4 in total) Namespace Name CPU Requests CPU Limits Memory Requests Memory Limits AGE --------- ---- ------------ ---------- --------------- ------------- --- kube-system kube-apiserver-host-10-10-18-46 250m (6%) 0 (0%) 0 (0%) 0 (0%) 169m kube-system kube-controller-manager-host-10-10-18-46 200m (5%) 0 (0%) 0 (0%) 0 (0%) 170m kube-system kube-proxy-zl8fw 0 (0%) 0 (0%) 0 (0%) 0 (0%) 166m kube-system kube-scheduler-host-10-10-18-46 100m (2%) 0 (0%) 0 (0%) 0 (0%) 170m Allocated resources: (Total limits may be over 100 percent, i.e., overcommitted.) Resource Requests Limits -------- -------- ------ cpu 550m (13%) 0 (0%) memory 0 (0%) 0 (0%) ephemeral-storage 0 (0%) 0 (0%) hugepages-2Mi 0 (0%) 0 (0%) hugepages-512Mi 0 (0%) 0 (0%) Events: Type Reason Age From Message ---- ------ ---- ---- ------- Normal Starting 171m kubelet, host-10-10-18-46 Starting kubelet. Normal NodeAllocatableEnforced 171m kubelet, host-10-10-18-46 Updated Node Allocatable limit across pods Normal NodeHasSufficientMemory 171m (x5 over 171m) kubelet, host-10-10-18-46 Node host-10-10-18-46 status is now: NodeHasSufficientMemory Normal NodeHasNoDiskPressure 171m (x4 over 171m) kubelet, host-10-10-18-46 Node host-10-10-18-46 status is now: NodeHasNoDiskPressure Normal NodeHasSufficientPID 171m (x4 over 171m) kubelet, host-10-10-18-46 Node host-10-10-18-46 status is now: NodeHasSufficientPID Normal Starting 170m kubelet, host-10-10-18-46 Starting kubelet. Normal NodeHasSufficientMemory 170m kubelet, host-10-10-18-46 Node host-10-10-18-46 status is now: NodeHasSufficientMemory Normal NodeHasNoDiskPressure 170m kubelet, host-10-10-18-46 Node host-10-10-18-46 status is now: NodeHasNoDiskPressure Normal NodeHasSufficientPID 170m kubelet, host-10-10-18-46 Node host-10-10-18-46 status is now: NodeHasSufficientPID Normal NodeAllocatableEnforced 170m kubelet, host-10-10-18-46 Updated Node Allocatable limit across pods Normal Starting 166m kube-proxy, host-10-10-18-46 Starting kube-proxy. Normal Starting 34m kubelet, host-10-10-18-46 Starting kubelet. Normal NodeAllocatableEnforced 34m kubelet, host-10-10-18-46 Updated Node Allocatable limit across pods Normal NodeHasSufficientMemory 34m (x8 over 34m) kubelet, host-10-10-18-46 Node host-10-10-18-46 status is now: NodeHasSufficientMemory Normal NodeHasNoDiskPressure 34m (x8 over 34m) kubelet, host-10-10-18-46 Node host-10-10-18-46 status is now: NodeHasNoDiskPressure Normal NodeHasSufficientPID 34m (x7 over 34m) kubelet, host-10-10-18-46 Node host-10-10-18-46 status is now: NodeHasSufficientPID Normal Starting 10m kubelet, host-10-10-18-46 Starting kubelet. Normal NodeAllocatableEnforced 10m kubelet, host-10-10-18-46 Updated Node Allocatable limit across pods Normal NodeHasSufficientMemory 10m (x8 over 10m) kubelet, host-10-10-18-46 Node host-10-10-18-46 status is now: NodeHasSufficientMemory Normal NodeHasNoDiskPressure 10m (x8 over 10m) kubelet, host-10-10-18-46 Node host-10-10-18-46 status is now: NodeHasNoDiskPressure Normal NodeHasSufficientPID 10m (x7 over 10m) kubelet, host-10-10-18-46 Node host-10-10-18-46 status is now: NodeHasSufficientPID [root@host-10-10-18-46 ~]# kubectl describe node host-10-10-18-46 | grep Taints Taints: node.kubernetes.io/not-ready:NoSchedule [root@host-10-10-18-46 ~]#
[root@host-10-10-18-46 ~]# kubectl taint node host-10-10-18-46 node-role.kubernetes.io/master:NoSchedule- error: taint "node-role.kubernetes.io/master:NoSchedule" not found [root@host-10-10-18-46 ~]# kubectl describe nodes |grep Taints Taints: node.kubernetes.io/not-ready:NoSchedule [root@host-10-10-18-46 ~]#
可以看到最后的方式为NoSchedule，可以通过kubectl taint命令进行设定如下三种方式，具体说明如下：
- NoSchedule: 不调度
- PreferNoSchedule: 尽量不调度
- NoExecute: 不调度并且立即驱逐节点上现存pod
[root@host-10-10-18-46 ~]# kubectl describe node host-10-10-18-46 Name: host-10-10-18-46 Roles: master Labels: beta.kubernetes.io/arch=arm64 beta.kubernetes.io/os=linux kubernetes.io/arch=arm64 kubernetes.io/hostname=host-10-10-18-46 kubernetes.io/os=linux node-role.kubernetes.io/master= Annotations: kubeadm.alpha.kubernetes.io/cri-socket: /var/run/dockershim.sock node.alpha.kubernetes.io/ttl: 0 volumes.kubernetes.io/controller-managed-attach-detach: true CreationTimestamp: Wed, 30 Jun 2021 12:03:35 +0800 Taints: node.kubernetes.io/not-ready:NoSchedule Unschedulable: false Lease: HolderIdentity: host-10-10-18-46 AcquireTime: <unset> RenewTime: Wed, 30 Jun 2021 14:54:05 +0800 Conditions: Type Status LastHeartbeatTime LastTransitionTime Reason Message ---- ------ ----------------- ------------------ ------ ------- MemoryPressure False Wed, 30 Jun 2021 14:49:22 +0800 Wed, 30 Jun 2021 12:03:35 +0800 KubeletHasSufficientMemory kubelet has sufficient memory available DiskPressure False Wed, 30 Jun 2021 14:49:22 +0800 Wed, 30 Jun 2021 12:03:35 +0800 KubeletHasNoDiskPressure kubelet has no disk pressure PIDPressure False Wed, 30 Jun 2021 14:49:22 +0800 Wed, 30 Jun 2021 12:03:35 +0800 KubeletHasSufficientPID kubelet has sufficient PID available Ready False Wed, 30 Jun 2021 14:49:22 +0800 Wed, 30 Jun 2021 12:03:35 +0800 KubeletNotReady runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized Addresses: InternalIP: 10.10.18.46 Hostname: host-10-10-18-46 Capacity: cpu: 4 ephemeral-storage: 7978Mi hugepages-2Mi: 0 hugepages-512Mi: 0 memory: 7756672Ki pods: 110 Allocatable: cpu: 4 ephemeral-storage: 7528985383 hugepages-2Mi: 0 hugepages-512Mi: 0 memory: 7654272Ki pods: 110 System Info: Machine ID: 30689d599b59462f9fee88051771bea5 System UUID: B80706BA-B199-4ED2-927B-66A6EC045417 Boot ID: 3205f1fc-6015-4fcd-a9c1-c9c24e2d8d80 Kernel Version: 4.14.0-115.el7a.0.1.aarch64 OS Image: CentOS Linux 7 (AltArch) Operating System: linux Architecture: arm64 Container Runtime Version: docker://20.10.7 Kubelet Version: v1.18.1 Kube-Proxy Version: v1.18.1 Non-terminated Pods: (4 in total) Namespace Name CPU Requests CPU Limits Memory Requests Memory Limits AGE --------- ---- ------------ ---------- --------------- ------------- --- kube-system kube-apiserver-host-10-10-18-46 250m (6%) 0 (0%) 0 (0%) 0 (0%) 169m kube-system kube-controller-manager-host-10-10-18-46 200m (5%) 0 (0%) 0 (0%) 0 (0%) 170m kube-system kube-proxy-zl8fw 0 (0%) 0 (0%) 0 (0%) 0 (0%) 166m kube-system kube-scheduler-host-10-10-18-46 100m (2%) 0 (0%) 0 (0%) 0 (0%) 170m Allocated resources: (Total limits may be over 100 percent, i.e., overcommitted.) Resource Requests Limits -------- -------- ------ cpu 550m (13%) 0 (0%) memory 0 (0%) 0 (0%) ephemeral-storage 0 (0%) 0 (0%) hugepages-2Mi 0 (0%) 0 (0%) hugepages-512Mi 0 (0%) 0 (0%) Events: Type Reason Age From Message ---- ------ ---- ---- ------- Normal Starting 171m kubelet, host-10-10-18-46 Starting kubelet. Normal NodeAllocatableEnforced 171m kubelet, host-10-10-18-46 Updated Node Allocatable limit across pods Normal NodeHasSufficientMemory 171m (x5 over 171m) kubelet, host-10-10-18-46 Node host-10-10-18-46 status is now: NodeHasSufficientMemory Normal NodeHasNoDiskPressure 171m (x4 over 171m) kubelet, host-10-10-18-46 Node host-10-10-18-46 status is now: NodeHasNoDiskPressure Normal NodeHasSufficientPID 171m (x4 over 171m) kubelet, host-10-10-18-46 Node host-10-10-18-46 status is now: NodeHasSufficientPID Normal Starting 170m kubelet, host-10-10-18-46 Starting kubelet. Normal NodeHasSufficientMemory 170m kubelet, host-10-10-18-46 Node host-10-10-18-46 status is now: NodeHasSufficientMemory Normal NodeHasNoDiskPressure 170m kubelet, host-10-10-18-46 Node host-10-10-18-46 status is now: NodeHasNoDiskPressure Normal NodeHasSufficientPID 170m kubelet, host-10-10-18-46 Node host-10-10-18-46 status is now: NodeHasSufficientPID Normal NodeAllocatableEnforced 170m kubelet, host-10-10-18-46 Updated Node Allocatable limit across pods Normal Starting 166m kube-proxy, host-10-10-18-46 Starting kube-proxy. Normal Starting 34m kubelet, host-10-10-18-46 Starting kubelet. Normal NodeAllocatableEnforced 34m kubelet, host-10-10-18-46 Updated Node Allocatable limit across pods Normal NodeHasSufficientMemory 34m (x8 over 34m) kubelet, host-10-10-18-46 Node host-10-10-18-46 status is now: NodeHasSufficientMemory Normal NodeHasNoDiskPressure 34m (x8 over 34m) kubelet, host-10-10-18-46 Node host-10-10-18-46 status is now: NodeHasNoDiskPressure Normal NodeHasSufficientPID 34m (x7 over 34m) kubelet, host-10-10-18-46 Node host-10-10-18-46 status is now: NodeHasSufficientPID Normal Starting 10m kubelet, host-10-10-18-46 Starting kubelet. Normal NodeAllocatableEnforced 10m kubelet, host-10-10-18-46 Updated Node Allocatable limit across pods Normal NodeHasSufficientMemory 10m (x8 over 10m) kubelet, host-10-10-18-46 Node host-10-10-18-46 status is now: NodeHasSufficientMemory Normal NodeHasNoDiskPressure 10m (x8 over 10m) kubelet, host-10-10-18-46 Node host-10-10-18-46 status is now: NodeHasNoDiskPressure Normal NodeHasSufficientPID 10m (x7 over 10m) kubelet, host-10-10-18-46 Node host-10-10-18-46 status is now: NodeHasSufficientPID
runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized
安装flannel后解决
[root@host-10-10-18-46 pki]# kubectl get nodes NAME STATUS ROLES AGE VERSION host-10-10-18-46 Ready master 3h54m v1.18.1
[root@host-10-10-18-46 pki]# kubectl get pods -o wide -n kube-system NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES coredns-546565776c-v49kt 0/1 ContainerCreating 0 75m <none> host-10-10-18-46 <none> <none> coredns-546565776c-z5pq6 0/1 ContainerCreating 0 75m <none> host-10-10-18-46 <none> <none> kube-apiserver-host-10-10-18-46 1/1 Running 64 3h59m 10.10.18.46 host-10-10-18-46 <none> <none> kube-controller-manager-host-10-10-18-46 1/1 Running 51 3h59m 10.10.18.46 host-10-10-18-46 <none> <none> kube-flannel-ds-arm64-x7mnq 1/1 Running 13 43m 10.10.18.46 host-10-10-18-46 <none> <none> kube-proxy-zl8fw 1/1 Running 0 3h55m 10.10.18.46 host-10-10-18-46 <none> <none> kube-scheduler-host-10-10-18-46 1/1 Running 52 3h59m 10.10.18.46 host-10-10-18-46 <none> <none>
加入其它master
token没有过期
如果没有--discovery-token-ca-cert-hash值，也可以通过以下命令获取
openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex | sed 's/^.* //'
-
如果是过期了，需要重新生成
1. 执行kubeadm token create --print-join-command，重新生成，重新生成基础的 join 命令（对于添加 master 节点还需要重新生成certificate-key，见下一步）
# 如果是添加 worker 节点，不需要执行这一步，直接使用上面返回的 join 命令加入集群。
2. 使用 kubeadm init phase upload-certs --experimental-upload-certs 重新生成certificate-key
# 添加 master 节点：用上面第1步生成的 join 命令和第2步生成的--certificate-key 值拼接起来执行 新增Master节点额外需要certificate-key参数，使用以下命令生成：
# 生成certificate-key
kubeadm init phase upload-certs --upload-certs
# 使用Node节点的join命令并且拼上--control-plane --certificate-key参数
kubeadm join kubernetes-vip:9443 --token bayqt8.eaafmfthasquy4yn --discovery-token-ca-cert-hash sha256:250115fad0a4b6852a919dbba4222ac65bc64843c660363ab119606ff8819d0a --control-plane --certificate-key bfd5bc7ff4aa54e1cba9a5979210c06ae087ae6fb9979af8f851554638889d7b[root@host-10-10-18-46 ~]# kubeadm token list TOKEN TTL EXPIRES USAGES DESCRIPTION EXTRA GROUPS abcdef.0123456789abcdef 20h 2021-07-01T12:03:42+08:00 authentication,signing <none> system:bootstrappers:kubeadm:default-node-token [root@host-10-10-18-46 ~]# openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex | sed 's/^.* //' 9bc0bcddb2b97791717943b714ffa410cb5963061889086f04eda6150cb590fc [root@host-10-10-18-46 ~]# kubeadm init phase upload-certs --upload-certs I0630 15:34:33.032985 8128 version.go:252] remote version is much newer: v1.21.2; falling back to: stable-1.18 W0630 15:34:34.097393 8128 configset.go:202] WARNING: kubeadm cannot validate component configs for API groups [kubelet.config.k8s.io kubeproxy.config.k8s.io] [upload-certs] Storing the certificates in Secret "kubeadm-certs" in the "kube-system" Namespace [upload-certs] Using certificate key: 8c94eb58dfdfc88b2f949d59f7f4348984dc0b155e37488a2f95df7048ca7374
[root@host-10-10-18-46 ~]# kubeadm token list TOKEN TTL EXPIRES USAGES DESCRIPTION EXTRA GROUPS abcdef.0123456789abcdef 20h 2021-07-01T12:03:42+08:00 authentication,signing <none> system:bootstrappers:kubeadm:default-node-token s7li41.5u78f4i2oqfg4t1c 1h 2021-06-30T17:34:43+08:00 <none> Proxy for managing TTL for the kubeadm-certs secret <none> [root@host-10-10-18-46 ~]#
[root@host-10-10-18-46 ~]# openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex | sed 's/^.* //' 9bc0bcddb2b97791717943b714ffa410cb5963061889086f04eda6150cb590fc [root@host-10-10-18-46 ~]#
拷贝秘钥
scp ca.* sa.* front-proxy-ca.* root@10.10.18.45:/etc/kubernetes/pki/