基于kubeadm的etcd单节点扩容

基于kubeadm的etcd单节点扩容

签发证书

/opt# cd ~/openssl/
~/openssl# cp /etc/kubernetes/pki/etcd/ca.crt .
~/openssl# cp /etc/kubernetes/pki/etcd/ca.key .

 证书签发

~/openssl# vi server.cnf

[ req ]
req_extensions      = v3_req
distinguished_name  = req_distinguished_name
[req_distinguished_name]
[ v3_req ]
basicConstraints    = CA:FALSE
extendedKeyUsage    = clientAuth, serverAuth
keyUsage            = nonRepudiation, digitalSignature, keyEncipherment
subjectAltName      = @alt_names
[alt_names]
IP.1 = 10.53.5.165
IP.2 = 10.53.4.221
IP.3 = 10.53.6.90


~/openssl# openssl genrsa -out server.key 4096
~/openssl# openssl req -new -key server.key -out server.csr -subj "/CN=10.53.5.165" -config server.cnf
~/openssl# openssl x509 -req -in server.csr -CA ca.crt 
        -CAkey ca.key -CAcreateserial 
        -out server.crt -days 1825 
        -extfile server.cnf -extensions v3_req
        
~/openssl# vi peer.cnf 

[ req ]
req_extensions     = v3_req
distinguished_name = req_distinguished_name

[req_distinguished_name]

[ v3_req ]
extendedKeyUsage   = clientAuth, serverAuth
keyUsage           = critical, digitalSignature, keyEncipherment
subjectAltName     = @alt_names

[alt_names]
IP.1 = 10.53.5.165
IP.2 = 10.53.4.221
IP.3 = 10.53.6.90

~/openssl# openssl genrsa -out peer.key 4096
~/openssl# openssl req -new -key peer.key -out peer.csr 
        -subj "/CN=10.53.5.165" 
        -config peer.cnf
~/openssl# openssl x509 -req -in peer.csr 
        -CA ca.crt -CAkey ca.key -CAcreateserial 
        -out peer.crt -days 1825 
        -extfile peer.cnf -extensions v3_req


~/openssl# vi client.cnf	
[ req ]
req_extensions     = v3_req
distinguished_name = req_distinguished_name

[req_distinguished_name]

[ v3_req ]
extendedKeyUsage   = clientAuth
keyUsage           = critical, digitalSignature, keyEncipherment

~/openssl# openssl genrsa -out apiserver-etcd-client.key 4096

~/openssl# openssl req -new -key apiserver-etcd-client.key -out client.csr 
        -subj "/CN=10.53.5.165" 
        -config client.cnf

~/openssl# openssl x509 -req -in client.csr 
        -CA ca.crt -CAkey ca.key -CAcreateserial 
        -out apiserver-etcd-client.crt -days 1825 
        -extfile client.cnf -extensions v3_req

扩容第二个节点

将证书拷贝到其他节点

~/openssl# scp -i diamond.yaml -r ~/openssl ubuntu@10.53.4.221:/home/ubuntu

 将证书拷贝到etcd目录下

/home/ubuntu/openssl# mkdir /etc/kubernetes/pki/etcd
/home/ubuntu/openssl# cp ca.crt ca.key peer.crt peer.key server.crt server.key /etc/kubernetes/pki/etcd/

 编辑etcd.yaml

/etc/kubernetes/manifests# systemctl stop kubelet

    - --advertise-client-urls=https://10.53.4.221:2379

    - --initial-advertise-peer-urls=https://10.53.4.221:2380

    - --initial-cluster=wangshile-vendor-4-10.53.5.165=https://10.53.5.165:2380,bj-idc1-10-53-4-221-10.53.4.221=https://10.53.4.221:2380

    - --initial-cluster-state=existing

    - --listen-client-urls=https://127.0.0.1:2379,https://10.53.4.221:2379

    - --listen-peer-urls=https://10.53.4.221:2380

    - --name=bj-idc1-10-53-4-221-10.53.4.221

/etc/kubernetes/pki/etcd# cd /etc/kubernetes/manifests/
/etc/kubernetes/manifests# docker ps -a | grep etcd

 主节点member add添加成员（千万不要先启动kubelet）

~# docker run --rm --net=host -v '/etc/kubernetes/pki/etcd:/etc/kubernetes/pki/etcd' --env ETCDCTL_API=3 -v '/var/lib/etcd:/var/lib/etcd' 'registry.sensetime.com/diamond/etcd:3.3.10' /bin/sh -c "etcdctl  --endpoints=https://10.53.5.165:2379 --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key --cacert=/etc/kubernetes/pki/etcd/ca.crt member list"

~# docker run --rm --net=host -v '/etc/kubernetes/pki/etcd:/etc/kubernetes/pki/etcd'  --env ETCDCTL_API=3 -v '/var/lib/etcd:/var/lib/etcd' 'registry.sensetime.com/diamond/etcd:3.3.10' /bin/sh -c "etcdctl --endpoints=https://10.53.5.165:2379 --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key --cacert=/etc/kubernetes/pki/etcd/ca.crt  member add bj-idc1-10-53-4-221-10.53.4.221 --peer-urls='https://10.53.4.221:2380'"

1241287698e4bb77, unstarted, , https://10.53.4.221:2380, 
8e9e05c52164694d, started, wangshile-vendor-4-10.53.5.165, https://10.53.5.165:2380, https://10.53.5.165:2379

这时候单节点集群会出现不可用状态

启动新节点，等待kubelet自动拉起pod

/etc/kubernetes/manifests# systemctl start kubelet
/etc/kubernetes/manifests# docker ps -a | grep etcd
/etc/kubernetes/manifests# netstat -tnlp| grep etcd 
tcp        0      0 127.0.0.1:2379          0.0.0.0:*               LISTEN      9134/etcd       
tcp        0      0 10.53.4.221:2379        0.0.0.0:*               LISTEN      9134/etcd       
tcp        0      0 10.53.4.221:2380        0.0.0.0:*               LISTEN      9134/etcd 

 当前节点查看

~# docker run --rm --net=host -v '/etc/kubernetes/pki/etcd:/etc/kubernetes/pki/etcd' --env ETCDCTL_API=3  'registry.sensetime.com/diamond/etcd:3.3.10' /bin/sh -c "etcdctl  --endpoints=https://10.53.4.221:2379 --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key --cacert=/etc/kubernetes/pki/etcd/ca.crt member list"
1241287698e4bb77, started, bj-idc1-10-53-4-221-10.53.4.221, https://10.53.4.221:2380, https://10.53.4.221:2379
8e9e05c52164694d, started, wangshile-vendor-4-10.53.5.165, https://10.53.5.165:2380, https://10.53.5.165:2379

# 查看集群健康状态
~# docker run --rm --net=host -v '/etc/kubernetes/pki/etcd:/etc/kubernetes/pki/etcd' 'registry.sensetime.com/diamond/etcd:3.3.10' /bin/sh -c "etcdctl  --endpoints=https://10.53.4.221:2379 --cert-file=/etc/kubernetes/pki/etcd/server.crt --key-file=/etc/kubernetes/pki/etcd/server.key --ca-file=/etc/kubernetes/pki/etcd/ca.crt cluster-health"
member 1241287698e4bb77 is healthy: got healthy result from https://10.53.4.221:2379
member 8e9e05c52164694d is healthy: got healthy result from https://10.53.5.165:2379
cluster is healthy

# 查看pod 
/etc/kubernetes/manifests# ll -h /var/lib/etcd/member/snap/ 
~# kubectl -n kube-system get po| grep etcd 
etcd-test-bj-idc1-10-53-4-221-10.53.4.221                1/1       Running   0          3m46s
etcd-wangshile-vendor-4-10.53.5.165                      1/1       Running   6          6d16h

 扩容第三个节点

/etc/kubernetes/manifests# scp -i ~/diamond.yaml -r ~/openssl ubuntu@10.53.6.90:/home/ubuntu
/home/ubuntu/openssl# mkdir /etc/kubernetes/pki/etcd
/opt# cd /home/ubuntu/openssl/
/home/ubuntu/openssl# cp ca.crt ca.key peer.crt peer.key server.crt server.key /etc/kubernetes/pki/etcd/

 编辑etcd.yaml

/etc/kubernetes/manifests# systemctl stop kubelet
/etc/kubernetes/pki/etcd# cd /etc/kubernetes/manifests/
/etc/kubernetes/manifests# docker ps -a | grep etcd

 添加成员

~# docker run --rm --net=host -v '/etc/kubernetes/pki/etcd:/etc/kubernetes/pki/etcd' --env ETCDCTL_API=3 -v '/var/lib/etcd:/var/lib/etcd' 'registry.sensetime.com/diamond/etcd:3.3.10' /bin/sh -c "etcdctl  --endpoints=https://10.53.6.90:2379 --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key --cacert=/etc/kubernetes/pki/etcd/ca.crt member list"

~# docker run --rm --net=host -v '/etc/kubernetes/pki/etcd:/etc/kubernetes/pki/etcd'  --env ETCDCTL_API=3 -v '/var/lib/etcd:/var/lib/etcd' 'registry.sensetime.com/diamond/etcd:3.3.10' /bin/sh -c "etcdctl --endpoints=https://10.53.5.165:2379 --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key --cacert=/etc/kubernetes/pki/etcd/ca.crt  member add bj-idc1-10-53-6-90-10.53.6.90 --peer-urls='https://10.53.6.90:2380'"
1241287698e4bb77, unstarted, , https://10.53.4.221:2380, 
8e9e05c52164694d, started, wangshile-vendor-4-10.53.5.165, https://10.53.5.165:2380, https://10.53.5.165:2379

 启动新节点

/etc/kubernetes/manifests# systemctl start kubelet
/etc/kubernetes/manifests# docker ps -a | grep etcd
/etc/kubernetes/manifests# netstat -tnlp| grep etcd 
tcp        0      0 127.0.0.1:2379          0.0.0.0:*               LISTEN      9134/etcd       
tcp        0      0 10.53.4.221:2379        0.0.0.0:*               LISTEN      9134/etcd       
tcp        0      0 10.53.4.221:2380        0.0.0.0:*               LISTEN      9134/etcd 

~# docker run --rm --net=host -v '/etc/kubernetes/pki/etcd:/etc/kubernetes/pki/etcd' 'registry.sensetime.com/diamond/etcd:3.3.10' /bin/sh -c "etcdctl  --endpoints=https://10.53.4.221:2379 --cert-file=/etc/kubernetes/pki/etcd/server.crt --key-file=/etc/kubernetes/pki/etcd/server.key --ca-file=/etc/kubernetes/pki/etcd/ca.crt cluster-health"
member 1241287698e4bb77 is healthy: got healthy result from https://10.53.4.221:2379
member 5a4d54cb656c6a3c is healthy: got healthy result from https://10.53.6.90:2379
member 8e9e05c52164694d is healthy: got healthy result from https://10.53.5.165:2379
cluster is healthy

~# kubectl -n kube-system get po| grep etcd