基于kubeadm的etcd单节点扩容
签发证书
/opt# cd ~/openssl/ ~/openssl# cp /etc/kubernetes/pki/etcd/ca.crt . ~/openssl# cp /etc/kubernetes/pki/etcd/ca.key .
证书签发
~/openssl# vi server.cnf
[ req ]
req_extensions = v3_req
distinguished_name = req_distinguished_name
[req_distinguished_name]
[ v3_req ]
basicConstraints = CA:FALSE
extendedKeyUsage = clientAuth, serverAuth
keyUsage = nonRepudiation, digitalSignature, keyEncipherment
subjectAltName = @alt_names
[alt_names]
IP.1 = 10.53.5.165
IP.2 = 10.53.4.221
IP.3 = 10.53.6.90
~/openssl# openssl genrsa -out server.key 4096
~/openssl# openssl req -new -key server.key -out server.csr -subj "/CN=10.53.5.165" -config server.cnf
~/openssl# openssl x509 -req -in server.csr -CA ca.crt
-CAkey ca.key -CAcreateserial
-out server.crt -days 1825
-extfile server.cnf -extensions v3_req
~/openssl# vi peer.cnf
[ req ]
req_extensions = v3_req
distinguished_name = req_distinguished_name
[req_distinguished_name]
[ v3_req ]
extendedKeyUsage = clientAuth, serverAuth
keyUsage = critical, digitalSignature, keyEncipherment
subjectAltName = @alt_names
[alt_names]
IP.1 = 10.53.5.165
IP.2 = 10.53.4.221
IP.3 = 10.53.6.90
~/openssl# openssl genrsa -out peer.key 4096
~/openssl# openssl req -new -key peer.key -out peer.csr
-subj "/CN=10.53.5.165"
-config peer.cnf
~/openssl# openssl x509 -req -in peer.csr
-CA ca.crt -CAkey ca.key -CAcreateserial
-out peer.crt -days 1825
-extfile peer.cnf -extensions v3_req
~/openssl# vi client.cnf
[ req ]
req_extensions = v3_req
distinguished_name = req_distinguished_name
[req_distinguished_name]
[ v3_req ]
extendedKeyUsage = clientAuth
keyUsage = critical, digitalSignature, keyEncipherment
~/openssl# openssl genrsa -out apiserver-etcd-client.key 4096
~/openssl# openssl req -new -key apiserver-etcd-client.key -out client.csr
-subj "/CN=10.53.5.165"
-config client.cnf
~/openssl# openssl x509 -req -in client.csr
-CA ca.crt -CAkey ca.key -CAcreateserial
-out apiserver-etcd-client.crt -days 1825
-extfile client.cnf -extensions v3_req
扩容第二个节点
将证书拷贝到其他节点
~/openssl# scp -i diamond.yaml -r ~/openssl ubuntu@10.53.4.221:/home/ubuntu
将证书拷贝到etcd目录下
/home/ubuntu/openssl# mkdir /etc/kubernetes/pki/etcd /home/ubuntu/openssl# cp ca.crt ca.key peer.crt peer.key server.crt server.key /etc/kubernetes/pki/etcd/
编辑etcd.yaml
/etc/kubernetes/manifests# systemctl stop kubelet
- --advertise-client-urls=https://10.53.4.221:2379
- --initial-advertise-peer-urls=https://10.53.4.221:2380
- --initial-cluster=wangshile-vendor-4-10.53.5.165=https://10.53.5.165:2380,bj-idc1-10-53-4-221-10.53.4.221=https://10.53.4.221:2380
- --initial-cluster-state=existing
- --listen-client-urls=https://127.0.0.1:2379,https://10.53.4.221:2379
- --listen-peer-urls=https://10.53.4.221:2380
- --name=bj-idc1-10-53-4-221-10.53.4.221
/etc/kubernetes/pki/etcd# cd /etc/kubernetes/manifests/ /etc/kubernetes/manifests# docker ps -a | grep etcd
主节点member add添加成员（千万不要先启动kubelet）
~# docker run --rm --net=host -v '/etc/kubernetes/pki/etcd:/etc/kubernetes/pki/etcd' --env ETCDCTL_API=3 -v '/var/lib/etcd:/var/lib/etcd' 'registry.sensetime.com/diamond/etcd:3.3.10' /bin/sh -c "etcdctl --endpoints=https://10.53.5.165:2379 --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key --cacert=/etc/kubernetes/pki/etcd/ca.crt member list" ~# docker run --rm --net=host -v '/etc/kubernetes/pki/etcd:/etc/kubernetes/pki/etcd' --env ETCDCTL_API=3 -v '/var/lib/etcd:/var/lib/etcd' 'registry.sensetime.com/diamond/etcd:3.3.10' /bin/sh -c "etcdctl --endpoints=https://10.53.5.165:2379 --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key --cacert=/etc/kubernetes/pki/etcd/ca.crt member add bj-idc1-10-53-4-221-10.53.4.221 --peer-urls='https://10.53.4.221:2380'" 1241287698e4bb77, unstarted, , https://10.53.4.221:2380, 8e9e05c52164694d, started, wangshile-vendor-4-10.53.5.165, https://10.53.5.165:2380, https://10.53.5.165:2379
这时候单节点集群会出现不可用状态
启动新节点，等待kubelet自动拉起pod
/etc/kubernetes/manifests# systemctl start kubelet /etc/kubernetes/manifests# docker ps -a | grep etcd /etc/kubernetes/manifests# netstat -tnlp| grep etcd tcp 0 0 127.0.0.1:2379 0.0.0.0:* LISTEN 9134/etcd tcp 0 0 10.53.4.221:2379 0.0.0.0:* LISTEN 9134/etcd tcp 0 0 10.53.4.221:2380 0.0.0.0:* LISTEN 9134/etcd
当前节点查看
~# docker run --rm --net=host -v '/etc/kubernetes/pki/etcd:/etc/kubernetes/pki/etcd' --env ETCDCTL_API=3 'registry.sensetime.com/diamond/etcd:3.3.10' /bin/sh -c "etcdctl --endpoints=https://10.53.4.221:2379 --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key --cacert=/etc/kubernetes/pki/etcd/ca.crt member list" 1241287698e4bb77, started, bj-idc1-10-53-4-221-10.53.4.221, https://10.53.4.221:2380, https://10.53.4.221:2379 8e9e05c52164694d, started, wangshile-vendor-4-10.53.5.165, https://10.53.5.165:2380, https://10.53.5.165:2379 # 查看集群健康状态 ~# docker run --rm --net=host -v '/etc/kubernetes/pki/etcd:/etc/kubernetes/pki/etcd' 'registry.sensetime.com/diamond/etcd:3.3.10' /bin/sh -c "etcdctl --endpoints=https://10.53.4.221:2379 --cert-file=/etc/kubernetes/pki/etcd/server.crt --key-file=/etc/kubernetes/pki/etcd/server.key --ca-file=/etc/kubernetes/pki/etcd/ca.crt cluster-health" member 1241287698e4bb77 is healthy: got healthy result from https://10.53.4.221:2379 member 8e9e05c52164694d is healthy: got healthy result from https://10.53.5.165:2379 cluster is healthy # 查看pod /etc/kubernetes/manifests# ll -h /var/lib/etcd/member/snap/ ~# kubectl -n kube-system get po| grep etcd etcd-test-bj-idc1-10-53-4-221-10.53.4.221 1/1 Running 0 3m46s etcd-wangshile-vendor-4-10.53.5.165 1/1 Running 6 6d16h
/etc/kubernetes/manifests# scp -i ~/diamond.yaml -r ~/openssl ubuntu@10.53.6.90:/home/ubuntu /home/ubuntu/openssl# mkdir /etc/kubernetes/pki/etcd /opt# cd /home/ubuntu/openssl/ /home/ubuntu/openssl# cp ca.crt ca.key peer.crt peer.key server.crt server.key /etc/kubernetes/pki/etcd/
编辑etcd.yaml
/etc/kubernetes/manifests# systemctl stop kubelet /etc/kubernetes/pki/etcd# cd /etc/kubernetes/manifests/ /etc/kubernetes/manifests# docker ps -a | grep etcd
添加成员
~# docker run --rm --net=host -v '/etc/kubernetes/pki/etcd:/etc/kubernetes/pki/etcd' --env ETCDCTL_API=3 -v '/var/lib/etcd:/var/lib/etcd' 'registry.sensetime.com/diamond/etcd:3.3.10' /bin/sh -c "etcdctl --endpoints=https://10.53.6.90:2379 --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key --cacert=/etc/kubernetes/pki/etcd/ca.crt member list" ~# docker run --rm --net=host -v '/etc/kubernetes/pki/etcd:/etc/kubernetes/pki/etcd' --env ETCDCTL_API=3 -v '/var/lib/etcd:/var/lib/etcd' 'registry.sensetime.com/diamond/etcd:3.3.10' /bin/sh -c "etcdctl --endpoints=https://10.53.5.165:2379 --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key --cacert=/etc/kubernetes/pki/etcd/ca.crt member add bj-idc1-10-53-6-90-10.53.6.90 --peer-urls='https://10.53.6.90:2380'" 1241287698e4bb77, unstarted, , https://10.53.4.221:2380, 8e9e05c52164694d, started, wangshile-vendor-4-10.53.5.165, https://10.53.5.165:2380, https://10.53.5.165:2379
启动新节点
/etc/kubernetes/manifests# systemctl start kubelet /etc/kubernetes/manifests# docker ps -a | grep etcd /etc/kubernetes/manifests# netstat -tnlp| grep etcd tcp 0 0 127.0.0.1:2379 0.0.0.0:* LISTEN 9134/etcd tcp 0 0 10.53.4.221:2379 0.0.0.0:* LISTEN 9134/etcd tcp 0 0 10.53.4.221:2380 0.0.0.0:* LISTEN 9134/etcd ~# docker run --rm --net=host -v '/etc/kubernetes/pki/etcd:/etc/kubernetes/pki/etcd' 'registry.sensetime.com/diamond/etcd:3.3.10' /bin/sh -c "etcdctl --endpoints=https://10.53.4.221:2379 --cert-file=/etc/kubernetes/pki/etcd/server.crt --key-file=/etc/kubernetes/pki/etcd/server.key --ca-file=/etc/kubernetes/pki/etcd/ca.crt cluster-health" member 1241287698e4bb77 is healthy: got healthy result from https://10.53.4.221:2379 member 5a4d54cb656c6a3c is healthy: got healthy result from https://10.53.6.90:2379 member 8e9e05c52164694d is healthy: got healthy result from https://10.53.5.165:2379 cluster is healthy ~# kubectl -n kube-system get po| grep etcd