k8s 集群安装---其余master node

# master01上执行
USER=root # customizable
CONTROL_PLANE_IPS="10.3.65.18"
for host in ${CONTROL_PLANE_IPS}; do
    scp -P 10088 /etc/kubernetes/pki/ca.crt "${USER}"@$host:/tmp
    scp -P 10088 /etc/kubernetes/pki/ca.key "${USER}"@$host:/tmp
    scp -P 10088 /etc/kubernetes/pki/sa.key "${USER}"@$host:/tmp
    scp -P 10088 /etc/kubernetes/pki/sa.pub "${USER}"@$host:/tmp
    scp -P 10088 /etc/kubernetes/pki/front-proxy-ca.crt "${USER}"@$host:/tmp
    scp -P 10088 /etc/kubernetes/pki/front-proxy-ca.key "${USER}"@$host:/tmp
    scp -P 10088 /etc/kubernetes/pki/etcd/ca.crt "${USER}"@$host:/tmp/etcd-ca.crt
    # Quote this line if you are using external etcd
    scp -P 10088 /etc/kubernetes/pki/etcd/ca.key "${USER}"@$host:/tmp/etcd-ca.key
done

#其余master上执行
USER=tmp # customizable
mkdir -p /etc/kubernetes/pki/etcd
mv /${USER}/ca.crt /etc/kubernetes/pki/
mv /${USER}/ca.key /etc/kubernetes/pki/
mv /${USER}/sa.pub /etc/kubernetes/pki/
mv /${USER}/sa.key /etc/kubernetes/pki/
mv /${USER}/front-proxy-ca.crt /etc/kubernetes/pki/
mv /${USER}/front-proxy-ca.key /etc/kubernetes/pki/
mv /${USER}/etcd-ca.crt /etc/kubernetes/pki/etcd/ca.crt
# Quote this line if you are using external etcd
mv /${USER}/etcd-ca.key /etc/kubernetes/pki/etcd/ca.key

kubeadm config images pull --image-repository=registry.aliyuncs.com/google_containers

docker tag registry.aliyuncs.com/google_containers/kube-proxy:v1.17.17 k8s.gcr.io/kube-proxy:v1.17.17
docker tag registry.aliyuncs.com/google_containers/kube-apiserver:v1.17.17 k8s.gcr.io/kube-apiserver:v1.17.17
docker tag registry.aliyuncs.com/google_containers/kube-controller-manager:v1.17.17 k8s.gcr.io/kube-controller-manager:v1.17.17
docker tag registry.aliyuncs.com/google_containers/kube-scheduler:v1.17.17 k8s.gcr.io/kube-scheduler:v1.17.17
docker tag registry.aliyuncs.com/google_containers/coredns:1.6.5 k8s.gcr.io/coredns:1.6.5
docker tag registry.aliyuncs.com/google_containers/etcd:3.4.3-0 k8s.gcr.io/etcd:3.4.3-0
docker tag registry.aliyuncs.com/google_containers/pause:3.1 k8s.gcr.io/pause:3.1


#node加入命令
#kubeadm join 10.3.65.37:6443 --token m40u64.k37bumocdfo4gmjj     --discovery-token-ca-cert-hash sha256:d11a8901ec1459035cbd9265d6421438a16af110b506929a4a504809dd0676cf  

#master加入命令

kubeadm join 10.3.65.37:6443 --token e9kub4.1y1tsheeinjonyaz  --discovery-token-ca-cert-hash sha256:d11a8901ec1459035cbd9265d6421438a16af110b506929a4a504809dd0676cf --control-plane

初始化脚本，就那几个生成配置文件的，需要确认重改，别的都对

#master上操作
scp /etc/yum.repos.d/kubernetes.repo xgcloud-ops-k8s-cluster-3:/etc/yum.repos.d/
scp /etc/yum.repos.d/kubernetes.repo xgcloud-ops-k8s-cluster-2:/etc/yum.repos.d/
scp /etc/yum.repos.d/kubernetes.repo xgcloud-ops-k8s-cluster-1:/etc/yum.repos.d/






#node 上操作
#有互联网可以用chronyd服务，或者自己的NTP服务
systemctl status chronyd
systemctl restart chronyd

#加host
#hostnamectl set-hostname k8s-master（192.168.73.138主机打命令）
#hostnamectl set-hostname k8s-node01（192.168.73.139主机打命令）
#hostnamectl set-hostname k8s-node02 (192.168.73.140主机打命令)
cat >> /etc/hosts << EOF
10.3.65.37 xgcloud-ops-k8s-cluster-4
10.3.65.18 xgcloud-ops-k8s-cluster-3
10.3.65.14 xgcloud-ops-k8s-cluster-2
10.3.65.48 xgcloud-ops-k8s-cluster-1
EOF
 
#关闭iptables和firewalld服务

systemctl status firewalld
systemctl stop firewalld
systemctl stop iptables
systemctl disable firewalld
systemctl disable iptables

#内核参数永久修改
cat <<EOF >  /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
EOF

sysctl -p /etc/sysctl.d/k8s.conf

#SELINUX disable
getenforce
sed -i 's/^SELINUX=.*/SELINUX=disabled/' /etc/selinux/config  && setenforce 0

#禁用swap 
swapoff -a
sed -i '/ swap / s/^(.*)$/#1/g' /etc/fstab 

#启用ipvs内核模块 写进去吧，这样出错了

cat >> /etc/sysconfig/modules/ipvs.modules << EOF
#!/bin/bash
ipvs_mods_dir="/usr/lib/modules/$(uname -r)/kernel/net/netfilter/ipvs"
for mod in $(ls $ipvs_mods_dir|grep -o "^[^.]*");do
    /sbin/modinfo -F filename $mod &> /dev/null
    if [ $? -eq 0 ];then
        /sbin/modprobe $mod
    fi
done
EOF
###vi /etc/sysconfig/modules/ipvs.modules
###
####!/bin/bash
###ipvs_mods_dir="/usr/lib/modules/$(uname -r)/kernel/net/netfilter/ipvs"
###for mod in $(ls $ipvs_mods_dir|grep -o "^[^.]*");do
###    /sbin/modinfo -F filename $mod &> /dev/null
###    if [$? -eq 0 ];then
###        /sbin/modprobe $mod
###    fi
###done

chmod +x /etc/sysconfig/modules/ipvs.modules
bash /etc/sysconfig/modules/ipvs.modules


######安装docker###########
cd /etc/yum.repos.d/
wget http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
#scp到各个机器
yum install -y yum-utils   device-mapper-persistent-data   lvm2
yum -y install docker-ce

touch /etc/docker/daemon.json 

 cat > /etc/docker/daemon.json <<EOF
{
"exec-opts": ["native.cgroupdriver=systemd"],
"registry-mirrors": ["https://mirror.aliyuncs.com","https://mirror.baidubce.com", "http://docker.mirrors.ustc.edu.cn", "http://hub-mirror.c.163.com"],
"log-driver":"json-file",
"log-opts": {"max-file": "20","max-size": "100m"}
}
EOF

systemctl daemon-reload
systemctl restart docker
systemctl enable docker
 
 cat > /etc/yum.repos.d/kubernetes.repo <<EOF
[kubernetes]
name=Kubernetes Repo
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/
gpgcheck=1
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
　　　　https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg
enabled=1
EOF


#yum repolist
#yum list all | grep "^kube"
#yum list kubeadm --showduplicates  这个命令可以看到有哪些版本可以安装
#yum remove kubectl kubeadm kubelet
#安装自己需要的版本
yum -y install kubectl-1.17.3-0 kubeadm-1.17.3-0 kubelet-1.17.3-0
systemctl enable kubelet
 cat > /etc/sysconfig/kubelet <<EOF
KUBELET_EXTRA_ARGS="--fail-swap-on=false 
--runtime-cgroups=/systemd/system.slice 
--kubelet-cgroups=/systemd/system.slice 
--allowed-unsafe-sysctls 'kernel.msg*,net.core.somaxconn"
EOF

image:

# 查看需要的镜像
kubeadm config images list
    k8s.gcr.io/kube-apiserver:v1.18.5
    k8s.gcr.io/kube-controller-manager:v1.18.5
    k8s.gcr.io/kube-scheduler:v1.18.5
    k8s.gcr.io/kube-proxy:v1.18.5
    k8s.gcr.io/pause:3.2
    k8s.gcr.io/etcd:3.4.3-0
    k8s.gcr.io/coredns:1.6.7

docker pull daocloud.io/daocloud/kube-apiserver:v1.18.5
docker pull daocloud.io/daocloud/kube-controller-manager:v1.18.5
docker pull daocloud.io/daocloud/kube-scheduler:v1.18.5
docker pull daocloud.io/daocloud/kube-proxy:v1.18.5
docker pull daocloud.io/daocloud/pause:3.2
docker pull daocloud.io/daocloud/etcd:3.4.3-0
docker pull daocloud.io/daocloud/coredns:1.6.7

docker tag daocloud.io/daocloud/kube-apiserver:v1.18.5 k8s.gcr.io/kube-apiserver:v1.18.5
docker tag daocloud.io/daocloud/kube-controller-manager:v1.18.5 k8s.gcr.io/kube-controller-manager:v1.18.5
docker tag daocloud.io/daocloud/kube-scheduler:v1.18.5 k8s.gcr.io/kube-scheduler:v1.18.5
docker tag daocloud.io/daocloud/kube-proxy:v1.18.5 k8s.gcr.io/kube-proxy:v1.18.5
docker tag daocloud.io/daocloud/pause:3.2  k8s.gcr.io/pause:3.2
docker tag daocloud.io/daocloud/etcd:3.4.3-0  k8s.gcr.io/etcd:3.4.3-0
docker tag daocloud.io/daocloud/coredns:1.6.7  k8s.gcr.io/coredns:1.6.7

docker rmi daocloud.io/daocloud/kube-apiserver:v1.18.5       
docker rmi daocloud.io/daocloud/kube-controller-manager:v1.18.5
docker rmi daocloud.io/daocloud/kube-scheduler:v1.18.5
docker rmi daocloud.io/daocloud/kube-proxy:v1.18.5      
docker rmi daocloud.io/daocloud/pause:3.2       
docker rmi daocloud.io/daocloud/etcd:3.4.3-0           
docker rmi daocloud.io/daocloud/coredns:1.6.7 

#备注，可以只在一个节点上下载后save下来，然后copy到其他节点load。这样速度会快些
docker save -o kube-proxy_v1.13.0.tar k8s.gcr.io/kube-proxy:v1.13.0
docker save -o kube-apiserver_v1.13.0.tar k8s.gcr.io/kube-apiserver:v1.13.0
docker save -o kube-controller-manager_v1.13.0.tar k8s.gcr.io/kube-controller-manager:v1.13.0
docker save -o coredns_1.2.6.tar k8s.gcr.io/coredns:1.2.6
docker save -o etcd_3.2.24.tar k8s.gcr.io/etcd:3.2.24 
docker save -o pause_3.1.tar k8s.gcr.io/pause:3.1

scp to other nodes

for var in $(ls);do docker load < $var;done


docker images
REPOSITORY                           TAG                 IMAGE ID            CREATED             SIZE
k8s.gcr.io/kube-proxy                v1.18.5             a1daed4e2b60        12 days ago         117MB
k8s.gcr.io/kube-apiserver            v1.18.5             08ca24f16874        12 days ago         173MB
k8s.gcr.io/kube-controller-manager   v1.18.5             8d69eaf196dc        12 days ago         162MB
k8s.gcr.io/kube-scheduler            v1.18.5             39d887c6621d        12 days ago         95.3MB
k8s.gcr.io/pause                     3.2                 80d28bedfe5d        4 months ago        683kB
k8s.gcr.io/coredns                   1.6.7               67da37a9a360        5 months ago        43.8MB
k8s.gcr.io/etcd                      3.4.3-0             303ce5db0e90        8 months ago        288MB