一、系统初始化
系统初始化参考：https://www.cnblogs.com/hujinzhong/p/12251486.html
节点准备：
| 节点 | ip |
| master01 | 10.0.0.11 |
| master02 | 10.0.0.12 |
| master03 | 10.0.0.13 |
| node01 | 10.0.0.21 |
| node02 | 10.0.0.22 |
| k8s-vip | 10.0.0.100 |
以一台master节点为例：
1）主机名及hosts解析
[root@master01 ~]# hostnamectl set-hostname k8s-master01 [root@master01 ~]# vim /etc/hosts 127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 ::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 10.0.0.11 master01 10.0.0.12 master02 10.0.0.13 master03 10.0.0.21 node01 10.0.0.22 node02 10.0.0.100 k8s-vip
2）安装依赖包
[root@master01 ~]# curl -o /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo [root@master01 ~]# yum install -y conntrack ntpdate ntp ipvsadm ipset jq iptables curl sysstat libseccomp wget vim net-tools git
3）防火墙/swap/selinux
#防火墙 [root@master01 ~]# systemctl stop firewalld && systemctl disable firewalld [root@master01 ~]# yum install -y iptables-services && systemctl start iptables && systemctl enable iptables &&iptables -F && service iptables save #selinix [root@master01 ~]# setenforce 0 && sed -i 's#^SELINUX=.*#SELINUX=disabled#g' /etc/selinux/config setenforce: SELinux is disabled [root@master01 ~]# getenforce Disabled #关闭swap [root@master01 ~]# swapoff -a && sed -r -i '/swap/s@(.*)@#1@g' /etc/fstab
4）内核升级
[root@master01 ~]# rpm -Uvh http://www.elrepo.org/elrepo-release-7.0-3.el7.elrepo.noarch.rpm [root@master01 ~]# yum --enablerepo=elrepo-kernel install -y kernel-lt [root@master01 ~]# grub2-set-default "CentOS Linux (4.4.182-1.el7.elrepo.x86_64) 7 (Core)" [root@master01 ~]# reboot [root@k8s-master01 ~]# uname -r 4.4.213-1.el7.elrepo.x86_64
5）关闭NUMA
[root@k8s-master01 ~]# cp /etc/default/grub{,.bak}
[root@k8s-master01 ~]# vim /etc/default/grub # 在 GRUB_CMDLINE_LINUX 一行添加 `numa=off` 参数
GRUB_CMDLINE_LINUX="biosdevname=0 net.ifnames=0 rhgb quiet numa=off"
[root@k8s-master01 ~]# cp /boot/grub2/grub.cfg{,.bak}
[root@k8s-master01 ~]# grub2-mkconfig -o /boot/grub2/grub.cfg6）内核参数调整
cat > /etc/sysctl.d/kubernetes.conf <<EOF net.bridge.bridge-nf-call-iptables=1 net.bridge.bridge-nf-call-ip6tables=1 net.ipv4.ip_forward=1 net.ipv4.tcp_tw_recycle=0 vm.swappiness=0 # 禁止使用 swap 空间，只有当系统 OOM 时才允许使用它 vm.overcommit_memory=1 # 不检查物理内存是否够用 vm.panic_on_oom=0 # 开启OOM fs.inotify.max_user_instances=8192 fs.inotify.max_user_watches=1048576 fs.file-max=52706963 fs.nr_open=52706963 net.ipv6.conf.all.disable_ipv6=1 net.netfilter.nf_conntrack_max=2310720 EOF [root@k8s-master01 ~]# sysctl -p /etc/sysctl.d/kubernetes.conf
7）时区调整
[root@k8s-master01 ~]# timedatectl set-timezone Asia/Shanghai [root@k8s-master01 ~]# timedatectl set-local-rtc 0 [root@k8s-master01 ~]# systemctl restart rsyslog && systemctl restart crond
8）关闭不必要服务
[root@k8s-master01 ~]# systemctl stop postfix && systemctl disable postfix
9）设置 rsyslogd 和 systemd journald
[root@k8s-master01 ~]# mkdir /var/log/journal [root@k8s-master01 ~]# mkdir /etc/systemd/journald.conf.d cat > /etc/systemd/journald.conf.d/99-prophet.conf <<EOF [Journal] #持久化保存到磁盘 Storage=persistent # 压缩历史日志 Compress=yes SyncIntervalSec=5m RateLimitInterval=30s RateLimitBurst=1000 # 最大占用空间10G SystemMaxUse=10G # 单日志文件最大200M SystemMaxFileSize=200M # 日志保存时间 2 周 MaxRetentionSec=2week # 不将日志转发到 syslog ForwardToSyslog=no EOF [root@master03 ~]# systemctl restart systemd-journald
10）kube-proxy开启ipvs的前提设置
[root@k8s-master01 ~]# modprobe br_netfilter cat > /etc/sysconfig/modules/ipvs.modules <<EOF #!/bin/bash modprobe -- ip_vs modprobe -- ip_vs_rr modprobe -- ip_vs_wrr modprobe -- ip_vs_sh modprobe -- nf_conntrack_ipv4 EOF [root@k8s-master01 ~]# chmod 755 /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules &&lsmod | grep -e ip_vs -e nf_conntrack_ipv4
11）docker安装
[root@k8s-master01 ~]# yum install -y yum-utils device-mapper-persistent-data lvm2
[root@k8s-master01 ~]# yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo[root@k8s-master01 ~]# yum update -y && yum install -y docker-ce
[root@k8s-master01 ~]# mkdir /etc/docker -p
[root@k8s-master01 ~]# mkdir -p /etc/systemd/system/docker.service.d
cat > /etc/docker/daemon.json <<EOF
{
"exec-opts": ["native.cgroupdriver=systemd"],
"log-driver": "json-file",
"log-opts": {
"max-size": "100m"
}
}
EOF
[root@master03 ~]# systemctl daemon-reload && systemctl start docker && systemctl enable docker12）导入相关镜像
[root@k8s-master01 ~]# mkdir /root/k8s -p [root@k8s-master01 ~]# cd k8s/ [root@k8s-master01 k8s]# docker load -i keepalived.tar [root@k8s-master01 k8s]# docker load -i haproxy.tar [root@k8s-master01 k8s]# docker images REPOSITORY TAG IMAGE ID CREATED SIZE k8s.gcr.io/kube-apiserver v1.15.1 68c3eb07bfc3 6 months ago 207MB k8s.gcr.io/kube-scheduler v1.15.1 b0b3c4c404da 6 months ago 81.1MB k8s.gcr.io/kube-proxy v1.15.1 89a062da739d 6 months ago 82.4MB k8s.gcr.io/kube-controller-manager v1.15.1 d75082f1d121 6 months ago 159MB k8s.gcr.io/coredns 1.3.1 eb516548c180 12 months ago 40.3MB k8s.gcr.io/etcd 3.3.10 2c4adeb21b4f 14 months ago 258MB wise2c/keepalived-k8s latest 0ba6a7862982 20 months ago 14MB wise2c/haproxy-k8s latest fde31577093d 20 months ago 71.1MB k8s.gcr.io/pause 3.1 da86e6ba6ca1 2 years ago 742kB
二、启动haproxy容器
[root@k8s-master01 lb]# cat start-haproxy.sh
#!/bin/bash
MasterIP1=10.0.0.11
MasterIP2=10.0.0.12
MasterIP3=10.0.0.13
MasterPort=6443
docker run -d --restart=always --name HAProxy-K8S -p 6444:6444
-e MasterIP1=$MasterIP1
-e MasterIP2=$MasterIP2
-e MasterIP3=$MasterIP3
-e MasterPort=$MasterPort
-v /data/lb/etc/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg
wise2c/haproxy-k8s
[root@k8s-master01 lb]# cat /data/lb/etc/haproxy.cfg
global
log 127.0.0.1 local0
log 127.0.0.1 local1 notice
maxconn 4096
#chroot /usr/share/haproxy
#user haproxy
#group haproxy
daemon
defaults
log global
mode http
option httplog
option dontlognull
retries 3
option redispatch
timeout connect 5000
timeout client 50000
timeout server 50000
frontend stats-front
bind *:8081
mode http
default_backend stats-back
frontend fe_k8s_6444
bind *:6444
mode tcp
timeout client 1h
log global
option tcplog
default_backend be_k8s_6443
acl is_websocket hdr(Upgrade) -i WebSocket
acl is_websocket hdr_beg(Host) -i ws
backend stats-back
mode http
balance roundrobin
stats uri /haproxy/stats
stats auth pxcstats:secret
backend be_k8s_6443
mode tcp
timeout queue 1h
timeout server 1h
timeout connect 1h
log global
balance roundrobin
server rancher01 10.0.0.11:6443
[root@k8s-master01 lb]# ./start-haproxy.sh
[root@k8s-master01 lb]# netstat -lntp|grep 6444
tcp6 0 0 :::6444 :::* LISTEN 40073/docker-proxy三、启动keepalive容器
[root@k8s-master01 lb]# cat start-keepalived.sh
#!/bin/bash
VIRTUAL_IP=10.0.0.100
INTERFACE=eth0
NETMASK_BIT=24
CHECK_PORT=6444
RID=10
VRID=160
MCAST_GROUP=224.0.0.18
docker run -itd --restart=always --name=Keepalived-K8S
--net=host --cap-add=NET_ADMIN
-e VIRTUAL_IP=$VIRTUAL_IP
-e INTERFACE=$INTERFACE
-e CHECK_PORT=$CHECK_PORT
-e RID=$RID
-e VRID=$VRID
-e NETMASK_BIT=$NETMASK_BIT
-e MCAST_GROUP=$MCAST_GROUP
wise2c/keepalived-k8s
[root@k8s-master01 lb]# ./start-keepalived.sh
3792352f22407eed1962a213ec82b1f00935b55b951704064c86142998bc4594
[root@k8s-master01 lb]# ip addr show
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 00:0c:29:97:61:29 brd ff:ff:ff:ff:ff:ff
inet 10.0.0.11/24 brd 10.0.0.255 scope global eth0
valid_lft forever preferred_lft forever
inet 10.0.0.100/24 scope global secondary eth0
valid_lft forever preferred_lft forever
3: docker0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default
link/ether 02:42:9f:ca:92:e2 brd ff:ff:ff:ff:ff:ff
inet 172.17.0.1/16 brd 172.17.255.255 scope global docker0
valid_lft forever preferred_lft forever
5: veth91153da@if4: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue master docker0 state UP group default
link/ether 92:fb:b5:91:09:bf brd ff:ff:ff:ff:ff:ff link-netnsid 0四、kubeadm安装
cat <<EOF > /etc/yum.repos.d/kubernetes.repo [kubernetes] name=Kubernetes baseurl=http://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64 enabled=1 gpgcheck=0 repo_gpgcheck=0 gpgkey=http://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg http://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg EOF [root@k8s-master01 lb]# yum -y install kubeadm-1.15.1 kubectl-1.15.1 kubelet-1.15.1 [root@k8s-master01 lb]# systemctl enable kubelet.service
五、第一个主节点初始化
[root@k8s-master01 k8s]# kubeadm config print init-defaults > kubeadm-config.yaml
apiVersion: kubeadm.k8s.io/v1beta2
bootstrapTokens:
- groups:
- system:bootstrappers:kubeadm:default-node-token
token: abcdef.0123456789abcdef
ttl: 24h0m0s
usages:
- signing
- authentication
kind: InitConfiguration
localAPIEndpoint:
advertiseAddress: 10.0.0.11
bindPort: 6443
nodeRegistration:
criSocket: /var/run/dockershim.sock
name: k8s-master01
taints:
- effect: NoSchedule
key: node-role.kubernetes.io/master
---
apiServer:
timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta2
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controlPlaneEndpoint: "10.0.0.100:6444" #vip地址
controllerManager: {}
dns:
type: CoreDNS
etcd:
local:
dataDir: /var/lib/etcd
imageRepository: k8s.gcr.io
kind: ClusterConfiguration
kubernetesVersion: v1.15.1
networking:
dnsDomain: cluster.local
podSubnet: "10.244.0.0/16"
serviceSubnet: 10.96.0.0/12
scheduler: {}
#初始化主节点
[root@k8s-master01 k8s]# kubeadm init --config=kubeadm-config.yaml --experimental-upload-certs | tee kubeadm-init.log
Flag --experimental-upload-certs has been deprecated, use --upload-certs instead
[init] Using Kubernetes version: v1.15.1
[preflight] Running pre-flight checks
[WARNING SystemVerification]: this Docker version is not on the list of validated versions: 19.03.5. Latest validated version: 18.09
[WARNING Hostname]: hostname "k8s-master01" could not be reached
[WARNING Hostname]: hostname "k8s-master01": lookup k8s-master01 on 223.5.5.5:53: no such host
[preflight] Pulling images required for setting up a Kubernetes cluster
[preflight] This might take a minute or two, depending on the speed of your internet connection
[preflight] You can also perform this action in beforehand using 'kubeadm config images pull'
[kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[kubelet-start] Activating the kubelet service
[certs] Using certificateDir folder "/etc/kubernetes/pki"
[certs] Generating "ca" certificate and key
[certs] Generating "apiserver-kubelet-client" certificate and key
[certs] Generating "apiserver" certificate and key
[certs] apiserver serving cert is signed for DNS names [k8s-master01 kubernetes kubernetes.default kubernetes.default.svc kubernetes.default.svc.cluster.local] and IPs [10.96.0.1 10.0.0.11 10.0.0.100]
[certs] Generating "front-proxy-ca" certificate and key
[certs] Generating "front-proxy-client" certificate and key
[certs] Generating "etcd/ca" certificate and key
[certs] Generating "apiserver-etcd-client" certificate and key
[certs] Generating "etcd/server" certificate and key
[certs] etcd/server serving cert is signed for DNS names [k8s-master01 localhost] and IPs [10.0.0.11 127.0.0.1 ::1]
[certs] Generating "etcd/peer" certificate and key
[certs] etcd/peer serving cert is signed for DNS names [k8s-master01 localhost] and IPs [10.0.0.11 127.0.0.1 ::1]
[certs] Generating "etcd/healthcheck-client" certificate and key
[certs] Generating "sa" key and public key
[kubeconfig] Using kubeconfig folder "/etc/kubernetes"
[endpoint] WARNING: port specified in controlPlaneEndpoint overrides bindPort in the controlplane address
[kubeconfig] Writing "admin.conf" kubeconfig file
[endpoint] WARNING: port specified in controlPlaneEndpoint overrides bindPort in the controlplane address
[kubeconfig] Writing "kubelet.conf" kubeconfig file
[endpoint] WARNING: port specified in controlPlaneEndpoint overrides bindPort in the controlplane address
[kubeconfig] Writing "controller-manager.conf" kubeconfig file
[endpoint] WARNING: port specified in controlPlaneEndpoint overrides bindPort in the controlplane address
[kubeconfig] Writing "scheduler.conf" kubeconfig file
[control-plane] Using manifest folder "/etc/kubernetes/manifests"
[control-plane] Creating static Pod manifest for "kube-apiserver"
[control-plane] Creating static Pod manifest for "kube-controller-manager"
[control-plane] Creating static Pod manifest for "kube-scheduler"
[etcd] Creating static Pod manifest for local etcd in "/etc/kubernetes/manifests"
[wait-control-plane] Waiting for the kubelet to boot up the control plane as static Pods from directory "/etc/kubernetes/manifests". This can take up to 4m0s
[apiclient] All control plane components are healthy after 20.508717 seconds
[upload-config] Storing the configuration used in ConfigMap "kubeadm-config" in the "kube-system" Namespace
[kubelet] Creating a ConfigMap "kubelet-config-1.15" in namespace kube-system with the configuration for the kubelets in the cluster
[upload-certs] Storing the certificates in Secret "kubeadm-certs" in the "kube-system" Namespace
[upload-certs] Using certificate key:
40d156da637185cb06a0ca91613c49840db34b21e09192e648a16249b66005a3
[mark-control-plane] Marking the node k8s-master01 as control-plane by adding the label "node-role.kubernetes.io/master=''"
[mark-control-plane] Marking the node k8s-master01 as control-plane by adding the taints [node-role.kubernetes.io/master:NoSchedule]
[bootstrap-token] Using token: abcdef.0123456789abcdef
[bootstrap-token] Configuring bootstrap tokens, cluster-info ConfigMap, RBAC Roles
[bootstrap-token] configured RBAC rules to allow Node Bootstrap tokens to post CSRs in order for nodes to get long term certificate credentials
[bootstrap-token] configured RBAC rules to allow the csrapprover controller automatically approve CSRs from a Node Bootstrap Token
[bootstrap-token] configured RBAC rules to allow certificate rotation for all node client certificates in the cluster
[bootstrap-token] Creating the "cluster-info" ConfigMap in the "kube-public" namespace
[addons] Applied essential addon: CoreDNS
[endpoint] WARNING: port specified in controlPlaneEndpoint overrides bindPort in the controlplane address
[addons] Applied essential addon: kube-proxy
Your Kubernetes control-plane has initialized successfully!
To start using your cluster, you need to run the following as a regular user:
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
https://kubernetes.io/docs/concepts/cluster-administration/addons/
You can now join any number of the control-plane node running the following command on each as root:
kubeadm join 10.0.0.100:6444 --token abcdef.0123456789abcdef
--discovery-token-ca-cert-hash sha256:f54325af8cd8f83189b47814aed0f459ef29cd6aa90313d7f777ad5cacc88ddb
--control-plane --certificate-key 40d156da637185cb06a0ca91613c49840db34b21e09192e648a16249b66005a3
Please note that the certificate-key gives access to cluster sensitive data, keep it secret!
As a safeguard, uploaded-certs will be deleted in two hours; If necessary, you can use
"kubeadm init phase upload-certs --upload-certs" to reload certs afterward.
Then you can join any number of worker nodes by running the following on each as root:
kubeadm join 10.0.0.100:6444 --token abcdef.0123456789abcdef
--discovery-token-ca-cert-hash sha256:f54325af8cd8f83189b47814aed0f459ef29cd6aa90313d7f777ad5cacc88ddb
[root@k8s-master01 k8s]# mkdir -p $HOME/.kube
[root@k8s-master01 k8s]# sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
[root@k8s-master01 k8s]# sudo chown $(id -u):$(id -g) $HOME/.kube/config
[root@k8s-master01 k8s]# kubectl get node
NAME STATUS ROLES AGE VERSION
k8s-master01 NotReady master 2m28s v1.15.1
#安装flannel
[root@k8s-master01 k8s]# wget https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
[root@k8s-master01 k8s]# kubectl create -f kube-flannel.yml
[root@k8s-master01 k8s]# kubectl get pod -n kube-system
NAME READY STATUS RESTARTS AGE
coredns-5c98db65d4-m4c9s 0/1 ContainerCreating 0 11m
coredns-5c98db65d4-pp925 0/1 ContainerCreating 0 11m
etcd-k8s-master01 1/1 Running 0 11m
kube-apiserver-k8s-master01 1/1 Running 0 11m
kube-controller-manager-k8s-master01 1/1 Running 0 11m
kube-flannel-ds-amd64-7q7g4 1/1 Running 0 3m2s
kube-proxy-zpwfz 1/1 Running 0 11m
kube-scheduler-k8s-master01 1/1 Running 0 10m
[root@k8s-master01 k8s]# kubectl get node
NAME STATUS ROLES AGE VERSION
k8s-master01 Ready master 12m v1.15.1六、后续master节点初始化
kubeadm join 10.0.0.100:6444 --token abcdef.0123456789abcdef
--discovery-token-ca-cert-hash sha256:f54325af8cd8f83189b47814aed0f459ef29cd6aa90313d7f777ad5cacc88ddb
--control-plane --certificate-key 40d156da637185cb06a0ca91613c49840db34b21e09192e648a16249b66005a3
[preflight] Running pre-flight checks
[WARNING SystemVerification]: this Docker version is not on the list of validated versions: 19.03.5. Latest validated version: 18.09
[preflight] Reading configuration from the cluster...
[preflight] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -oyaml'
[preflight] Running pre-flight checks before initializing the new control plane instance
[preflight] Pulling images required for setting up a Kubernetes cluster
[preflight] This might take a minute or two, depending on the speed of your internet connection
[preflight] You can also perform this action in beforehand using 'kubeadm config images pull'
[download-certs] Downloading the certificates in Secret "kubeadm-certs" in the "kube-system" Namespace
[certs] Using certificateDir folder "/etc/kubernetes/pki"
[certs] Generating "front-proxy-client" certificate and key
[certs] Generating "etcd/server" certificate and key
[certs] etcd/server serving cert is signed for DNS names [master02 localhost] and IPs [10.0.0.12 127.0.0.1 ::1]
[certs] Generating "etcd/peer" certificate and key
[certs] etcd/peer serving cert is signed for DNS names [master02 localhost] and IPs [10.0.0.12 127.0.0.1 ::1]
[certs] Generating "etcd/healthcheck-client" certificate and key
[certs] Generating "apiserver-etcd-client" certificate and key
[certs] Generating "apiserver" certificate and key
[certs] apiserver serving cert is signed for DNS names [master02 kubernetes kubernetes.default kubernetes.default.svc kubernetes.default.svc.cluster.local] and IPs [10.96.0.1 10.0.0.12 10.0.0.100]
[certs] Generating "apiserver-kubelet-client" certificate and key
[certs] Valid certificates and keys now exist in "/etc/kubernetes/pki"
[certs] Using the existing "sa" key
[kubeconfig] Generating kubeconfig files
[kubeconfig] Using kubeconfig folder "/etc/kubernetes"
[endpoint] WARNING: port specified in controlPlaneEndpoint overrides bindPort in the controlplane address
[kubeconfig] Writing "admin.conf" kubeconfig file
[kubeconfig] Writing "controller-manager.conf" kubeconfig file
[kubeconfig] Writing "scheduler.conf" kubeconfig file
[control-plane] Using manifest folder "/etc/kubernetes/manifests"
[control-plane] Creating static Pod manifest for "kube-apiserver"
[control-plane] Creating static Pod manifest for "kube-controller-manager"
[control-plane] Creating static Pod manifest for "kube-scheduler"
[check-etcd] Checking that the etcd cluster is healthy
[kubelet-start] Downloading configuration for the kubelet from the "kubelet-config-1.15" ConfigMap in the kube-system namespace
[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
[kubelet-start] Activating the kubelet service
[kubelet-start] Waiting for the kubelet to perform the TLS Bootstrap...
[etcd] Announced new etcd member joining to the existing etcd cluster
[etcd] Wrote Static Pod manifest for a local etcd member to "/etc/kubernetes/manifests/etcd.yaml"
[etcd] Waiting for the new etcd member to join the cluster. This can take up to 40s
[upload-config] Storing the configuration used in ConfigMap "kubeadm-config" in the "kube-system" Namespace
[mark-control-plane] Marking the node master02 as control-plane by adding the label "node-role.kubernetes.io/master=''"
[mark-control-plane] Marking the node master02 as control-plane by adding the taints [node-role.kubernetes.io/master:NoSchedule]
This node has joined the cluster and a new control plane instance was created:
* Certificate signing request was sent to apiserver and approval was received.
* The Kubelet was informed of the new secure connection details.
* Control plane (master) label and taint were applied to the new node.
* The Kubernetes control plane instances scaled up.
* A new etcd member was added to the local/stacked etcd cluster.
To start administering your cluster from this node, you need to run the following as a regular user:
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
Run 'kubectl get nodes' to see this node join the cluster.
[root@master02 k8s]# mkdir -p $HOME/.kube
[root@master02 k8s]# sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
[root@master02 k8s]# sudo chown $(id -u):$(id -g) $HOME/.kube/config
[root@master02 k8s]# kubectl get node
NAME STATUS ROLES AGE VERSION
k8s-master01 Ready master 12h v1.15.1
master02 Ready master 2m12s v1.15.1
#同理master03也可以加入master集群
[root@master03 lb]# kubectl get node
NAME STATUS ROLES AGE VERSION
k8s-master01 Ready master 12h v1.15.1
master02 Ready master 7m37s v1.15.1
master03 Ready master 70s v1.15.1
[root@master03 lb]# kubectl get pod -n kube-system
NAME READY STATUS RESTARTS AGE
coredns-5c98db65d4-m4c9s 1/1 Running 0 12h
coredns-5c98db65d4-pp925 1/1 Running 8 12h
etcd-k8s-master01 1/1 Running 1 12h
etcd-master02 1/1 Running 0 7m32s
etcd-master03 1/1 Running 0 64s
kube-apiserver-k8s-master01 1/1 Running 7 12h
kube-apiserver-master02 1/1 Running 0 7m33s
kube-apiserver-master03 1/1 Running 0 65s
kube-controller-manager-k8s-master01 1/1 Running 11 12h
kube-controller-manager-master02 1/1 Running 0 7m33s
kube-controller-manager-master03 1/1 Running 0 65s
kube-flannel-ds-amd64-2mt7n 1/1 Running 0 66s
kube-flannel-ds-amd64-4lv5m 1/1 Running 0 7m33s
kube-flannel-ds-amd64-7q7g4 1/1 Running 1 12h
kube-proxy-7qk9g 1/1 Running 0 66s
kube-proxy-zh9jg 1/1 Running 0 7m33s
kube-proxy-zpwfz 1/1 Running 1 12h
kube-scheduler-k8s-master01 1/1 Running 11 12h
kube-scheduler-master02 1/1 Running 0 7m33s
kube-scheduler-master03 1/1 Running 0 65s七、修改haproxy配置文件
[root@k8s-master01 etc]# cat haproxy.cfg
global
log 127.0.0.1 local0
log 127.0.0.1 local1 notice
maxconn 4096
#chroot /usr/share/haproxy
#user haproxy
#group haproxy
daemon
defaults
log global
mode http
option httplog
option dontlognull
retries 3
option redispatch
timeout connect 5000
timeout client 50000
timeout server 50000
frontend stats-front
bind *:8081
mode http
default_backend stats-back
frontend fe_k8s_6444
bind *:6444
mode tcp
timeout client 1h
log global
option tcplog
default_backend be_k8s_6443
acl is_websocket hdr(Upgrade) -i WebSocket
acl is_websocket hdr_beg(Host) -i ws
backend stats-back
mode http
balance roundrobin
stats uri /haproxy/stats
stats auth pxcstats:secret
backend be_k8s_6443
mode tcp
timeout queue 1h
timeout server 1h
timeout connect 1h
log global
balance roundrobin
server rancher01 10.0.0.11:6443
server rancher02 10.0.0.12:6443
server rancher03 10.0.0.13:6443
[root@k8s-master01 etc]# docker rm -f HAProxy-K8S && bash /data/lb/start-haproxy.sh
HAProxy-K8S
b3386548e1e8777167fcf9e92ee348b182a3e4a59b0e2a13384ff13bc4619a61
#三台都需要八、修改配置
当关掉一台节点，使用kubectl get node有时会卡住，可以修改连接本地的apiserver
[root@master02 ~]# vim .kube/config server: https://10.0.0.12:6443 #3台都要修改
九、集群健康检查
#etcd集群状态
[root@k8s-master01 ~]# kubectl -n kube-system exec etcd-k8s-master01 -- etcdctl
> --endpoints=https://10.0.0.11:2379
> --ca-file=/etc/kubernetes/pki/etcd/ca.crt
> --cert-file=/etc/kubernetes/pki/etcd/server.crt
> --key-file=/etc/kubernetes/pki/etcd/server.key cluster-health
member 52c20770c3586df3 is healthy: got healthy result from https://10.0.0.12:2379
member a4ef9fb7d070299d is healthy: got healthy result from https://10.0.0.13:2379
member cebdf10928a06f3c is healthy: got healthy result from https://10.0.0.11:2379
cluster is healthy
#kube-controller-manager（一个工作，两个抑制）
[root@k8s-master01 ~]# kubectl get endpoints kube-controller-manager --namespace=kube-system -o yaml
apiVersion: v1
kind: Endpoints
metadata:
annotations:
control-plane.alpha.kubernetes.io/leader: '{"holderIdentity":"master02_4a856c96-ded5-4083-b19e-fbe1e4debb0a","leaseDurationSeconds":15,"acquireTime":"2020-02-08T02:03:20Z","renewTime":"2020-02-08T02:31:43Z","leaderTransitions":11}'
creationTimestamp: "2020-02-07T13:35:54Z"
name: kube-controller-manager
namespace: kube-system
resourceVersion: "5693"
selfLink: /api/v1/namespaces/kube-system/endpoints/kube-controller-manager
uid: cedd7e2d-68b9-4626-946b-9e1b458b322c
#kube-scheduler
[root@k8s-master01 ~]# kubectl get endpoints kube-scheduler --namespace=kube-system -o yaml
apiVersion: v1
kind: Endpoints
metadata:
annotations:
control-plane.alpha.kubernetes.io/leader: '{"holderIdentity":"master02_714ba13b-a72c-486e-b0cc-6fcfd688d215","leaseDurationSeconds":15,"acquireTime":"2020-02-08T02:03:21Z","renewTime":"2020-02-08T02:32:11Z","leaderTransitions":11}'
creationTimestamp: "2020-02-07T13:35:53Z"
name: kube-scheduler
namespace: kube-system
resourceVersion: "5738"
selfLink: /api/v1/namespaces/kube-system/endpoints/kube-scheduler
uid: 645b288f-0566-41f5-8c3f-d16ad4ae4171
------------------------------------------------------------------------------
[root@k8s-master01 ~]# kubectl get pod -n kube-system
NAME READY STATUS RESTARTS AGE
coredns-5c98db65d4-m4c9s 1/1 Running 1 12h
coredns-5c98db65d4-pp925 1/1 Running 9 12h
etcd-k8s-master01 1/1 Running 2 12h
etcd-master02 1/1 Running 0 30m
etcd-master03 1/1 Running 0 23m
kube-apiserver-k8s-master01 1/1 Running 8 12h
kube-apiserver-master02 1/1 Running 0 30m
kube-apiserver-master03 1/1 Running 0 23m
kube-controller-manager-k8s-master01 1/1 Running 12 12h
kube-controller-manager-master02 1/1 Running 0 30m
kube-controller-manager-master03 1/1 Running 0 23m
kube-flannel-ds-amd64-2mt7n 1/1 Running 0 23m
kube-flannel-ds-amd64-4lv5m 1/1 Running 0 30m
kube-flannel-ds-amd64-7q7g4 1/1 Running 2 12h
kube-proxy-7qk9g 1/1 Running 0 23m
kube-proxy-zh9jg 1/1 Running 0 30m
kube-proxy-zpwfz 1/1 Running 2 12h
kube-scheduler-k8s-master01 1/1 Running 12 12h
kube-scheduler-master02 1/1 Running 0 30m
kube-scheduler-master03 1/1 Running 0 23m
[root@k8s-master01 ~]# kubectl get node
NAME STATUS ROLES AGE VERSION
k8s-master01 Ready master 12h v1.15.1
master02 Ready master 30m v1.15.1
master03 Ready master 24m v1.15.1
十、node加入
直接执行初始化，node加入命令即可