十二、高可用master集群

1、部署一个master02

从master01上拷贝配置、安装、启动文件到master02上

[root@k8s-master01 ~]# scp -r /opt/kubernetes/ 10.16.8.151:/opt
[root@k8s-master01 ~]# scp -r /opt/etcd/ssl/ 10.16.8.151:/opt/etcd/ 
[root@k8s-master01 ~]# scp /usr/lib/systemd/system/{kube-apiserver,kube-controller-manager,kube-scheduler}.service 10.16.8.151:/usr/lib/systemd/system  

2、在master02上修改apiserver配置文件为本机IP：

[root@k8s-master02 ~]# cat /opt/kubernetes/cfg/kube-apiserver.conf 
KUBE_APISERVER_OPTS="--logtostderr=false 
--v=2 
--log-dir=/opt/kubernetes/logs 
--etcd-servers=https://10.16.8.161:2379,https://10.16.8.162:2379,https://10.16.8.163:2379 
--bind-address=10.16.8.151 
--secure-port=6443 
--advertise-address=10.16.8.151 
--allow-privileged=true 
--service-cluster-ip-range=10.0.0.0/24 
--enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota,NodeRestriction 
--authorization-mode=RBAC,Node 
--enable-bootstrap-token-auth=true 
--token-auth-file=/opt/kubernetes/cfg/token.csv 
--service-node-port-range=30000-32767 
--kubelet-client-certificate=/opt/kubernetes/ssl/server.pem 
--kubelet-client-key=/opt/kubernetes/ssl/server-key.pem 
--tls-cert-file=/opt/kubernetes/ssl/server.pem  
--tls-private-key-file=/opt/kubernetes/ssl/server-key.pem 
--client-ca-file=/opt/kubernetes/ssl/ca.pem 
--service-account-key-file=/opt/kubernetes/ssl/ca-key.pem 
--etcd-cafile=/opt/etcd/ssl/ca.pem 
--etcd-certfile=/opt/etcd/ssl/server.pem 
--etcd-keyfile=/opt/etcd/ssl/server-key.pem 
--audit-log-maxage=30 
--audit-log-maxbackup=3 
--audit-log-maxsize=100 
--audit-log-path=/opt/kubernetes/logs/k8s-audit.log"

3、启动

[root@k8s-master02 ~]# systemctl start kube-apiserver
[root@k8s-master02 ~]# systemctl start kube-controller-manager
[root@k8s-master02 ~]# systemctl start kube-scheduler
[root@k8s-master02 ~]# systemctl enable kube-apiserver
[root@k8s-master02 ~]# systemctl enable kube-controller-manager
[root@k8s-master02 ~]# systemctl enable kube-scheduler

4、查看master02

[root@k8s-master02 ~]# ln -s /opt/kubernetes/bin/kubectl /usr/local/bin/
[root@k8s-master02 ~]# kubectl get node
NAME         STATUS   ROLES    AGE     VERSION
k8s-node01   Ready    <none>   5h45m   v1.16.0
k8s-node02   Ready    <none>   5h42m   v1.16.0
k8s-node03   Ready    <none>   5h42m   v1.16.0

现在有两个Master了，下面我们在10.16.8.156上部署nginx，使用nginx的4层负债均衡功能来对master的apiserver进行高可用
1、安装部署nginx

[root@nginx02 ~]# yum install nginx
[root@nginx02 ~]# cat /etc/nginx/nginx.conf |egrep -v "^$|^#"
user nginx;
worker_processes auto;
error_log /var/log/nginx/error.log;
pid /run/nginx.pid;
include /usr/share/nginx/modules/*.conf;
events {
    worker_connections 1024;
}
stream {
    log_format  main  '$remote_addr $upstream_addr - [$time_local] $status $upstream_bytes_sent';
    access_log  /var/log/nginx/k8s-access.log  main;
    upstream k8s-apiserver {
                server 10.16.8.150:6443;
                server 10.16.8.151:6443;
            }
    
    server {
       listen 6443;
       proxy_pass k8s-apiserver;
    }
}
http {
    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';
    access_log  /var/log/nginx/access.log  main;
    sendfile            on;
    tcp_nopush          on;
    tcp_nodelay         on;
    keepalive_timeout   65;
    types_hash_max_size 2048;
    include             /etc/nginx/mime.types;
    default_type        application/octet-stream;
    include /etc/nginx/conf.d/*.conf;
}

[root@nginx02 ~]# systemctl start nginx
[root@nginx02 ~]# systemctl enable nginx

2、修改所有node的连接IP为10.16.8.156

[root@k8s-node01 opt]# cd /opt/kubernetes/cfg
[root@k8s-node01 cfg]# grep "10.16.8" * 
bootstrap.kubeconfig:    server: https://10.16.8.150:6443
kubelet.kubeconfig:    server: https://10.16.8.150:6443
kube-proxy.kubeconfig:    server: https://10.16.8.150:6443

[root@k8s-node01 cfg]# sed -i 's#10.16.8.150#10.16.8.156#g' *                          
[root@k8s-node01 cfg]# grep "10.16.8" *                      
bootstrap.kubeconfig:    server: https://10.16.8.156:6443
kubelet.kubeconfig:    server: https://10.16.8.156:6443
kube-proxy.kubeconfig:    server: https://10.16.8.156:6443

#重启k8s-node服务

[root@k8s-node02 ~]# systemctl restart kubelet
[root@k8s-node02 ~]# systemctl restart kube-proxy

3、测试

[root@k8s-node03 ~]# curl -k --header "Authorization: Bearer c47ffb939f5ca36231d9e3121a252940" https://10.16.8.156:6443/version 
{
  "major": "1",
  "minor": "16",
  "gitVersion": "v1.16.0",
  "gitCommit": "2bd9643cee5b3b3a5ecbd3af49d09018f0773c77",
  "gitTreeState": "clean",
  "buildDate": "2019-09-18T14:27:17Z",
  "goVersion": "go1.12.9",
  "compiler": "gc",
  "platform": "linux/amd64"
}

注：c47ffb939f5ca36231d9e3121a252940为部署master时生成的token