kubadm 安装k8s集群

 参考：https://blog.csdn.net/m350058411/article/details/105456953

https://www.kancloud.cn/pshizhsysu/kubernetes/2055662

 第一台master的整体脚本，脚本问题：

1. /etc/docker/daemon.json这个文件内容没有，手动弄下吧

2. /etc/sysconfig/modules/ipvs.modules 文件内容有问题，里面的命令执行了， 手动加进去，在执行下

3. kubeadm init 命令建议由yaml文件方式试试，这个是加的单master的命令，yaml配置例子：

　　kubeadm init --config kubeadm-config.yaml --upload-certs 

　　因为这里选择使用参数--upload-certs 所以【2. 同步证书到其他master节点】不在需要操作

apiVersion: kubeadm.k8s.io/v1beta2
kind: ClusterConfiguration
kubernetesVersion: v1.17.17
imageRepository: k8s.gcr.io
clusterName: lchuan-ceshi
certificatesDir: /etc/kubernetes/pki
apiServer:
  timeoutForControlPlane: 4m0s
  CertSANs:
  - "csapi.ejuops.com"
  - 10.3.65.37
  - 10.3.65.18
  - 10.3.65.14
controlPlaneEndpoint: "csapi.ejuops.com:6443"
dns:
  type: CoreDNS
networking:
  dnsDomain: cluster.local
  podSubnet: "192.168.0.0/16"
  serviceSubnet: 10.96.0.0/12

脚本：

#有互联网可以用chronyd服务，或者自己的NTP服务
systemctl status chronyd
systemctl restart chronyd

#加host
#hostnamectl set-hostname k8s-master（192.168.73.138主机打命令）
#hostnamectl set-hostname k8s-node01（192.168.73.139主机打命令）
#hostnamectl set-hostname k8s-node02 (192.168.73.140主机打命令)
cat >> /etc/hosts << EOF
10.3.65.37 xgcloud-ops-k8s-cluster-4
10.3.65.18 xgcloud-ops-k8s-cluster-3
10.3.65.14 xgcloud-ops-k8s-cluster-2
10.3.65.48 xgcloud-ops-k8s-cluster-1
EOF

#关闭iptables和firewalld服务

systemctl status firewalld
systemctl stop firewalld
systemctl stop iptables
systemctl disable firewalld
systemctl disable iptables

#内核参数永久修改
cat > /etc/sysctl.d/k8s.conf <<EOF 
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
EOF

sysctl -p /etc/sysctl.d/k8s.conf

#SELINUX disable
getenforce
sed -i 's/^SELINUX=.*/SELINUX=disabled/' /etc/selinux/config && setenforce 0

#禁用swap
swapoff -a
sed -i '/ swap / s/^(.*)$/#1/g' /etc/fstab

#启用ipvs内核模块 写进去吧，这样出错了
cat >> /etc/sysconfig/modules/ipvs.modules << EOF
#!/bin/bash
ipvs_mods_dir="/usr/lib/modules/$(uname -r)/kernel/net/netfilter/ipvs"
for mod in $(ls $ipvs_mods_dir|grep -o "^[^.]*");do
/sbin/modinfo -F filename $mod &> /dev/null
if [ $? -eq 0 ];then
/sbin/modprobe $mod
fi
done
EOF
###vi /etc/sysconfig/modules/ipvs.modules
###
####!/bin/bash
###ipvs_mods_dir="/usr/lib/modules/$(uname -r)/kernel/net/netfilter/ipvs"
###for mod in $(ls $ipvs_mods_dir|grep -o "^[^.]*");do
### /sbin/modinfo -F filename $mod &> /dev/null
### if [$? -eq 0 ];then
### /sbin/modprobe $mod
### fi
###done

chmod +x /etc/sysconfig/modules/ipvs.modules
bash /etc/sysconfig/modules/ipvs.modules

######安装docker###########
#yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
cd /etc/yum.repos.d/
wget http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
#scp到各个机器
yum install -y yum-utils device-mapper-persistent-data lvm2
yum -y install docker-ce

systemctl daemon-reload
systemctl restart docker
systemctl enable docker

cat > /etc/docker/daemon.json <<EOF
{
"exec-opts": ["native.cgroupdriver=systemd"],
"registry-mirrors": ["https://mirror.aliyuncs.com","https://mirror.baidubce.com", "http://docker.mirrors.ustc.edu.cn", "http://hub-mirror.c.163.com"],
"log-driver":"json-file",
"log-opts": {"max-file": "20","max-size": "100m"}
}
EOF

cat > /etc/yum.repos.d/kubernetes.repo <<EOF
[kubernetes]
name=Kubernetes Repo
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/
gpgcheck=1
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
　　　　https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg
enabled=1
EOF

#yum repolist
#yum list all | grep "^kube"
#yum list kubeadm --showduplicates 这个命令可以看到有哪些版本可以安装
#yum remove kubectl kubeadm kubelet
#安装自己需要的版本
yum -y install kubectl-1.17.3-0 kubeadm-1.17.3-0 kubelet-1.17.3-0

systemctl enable kubelet

#yum -y install kubectl kubeadm kubelet
rpm -ql kubelet
从阿里云镜像云下载kubeadm需要的镜像
#kubeadm config images pull --image-repository=registry.aliyuncs.com/google_containers

kubeadm config images pull --image-repository=registry.aliyuncs.com/google_containers

docker tag registry.aliyuncs.com/google_containers/kube-proxy:v1.17.17 k8s.gcr.io/kube-proxy:v1.17.17
docker tag registry.aliyuncs.com/google_containers/kube-apiserver:v1.17.17 k8s.gcr.io/kube-apiserver:v1.17.17
docker tag registry.aliyuncs.com/google_containers/kube-controller-manager:v1.17.17 k8s.gcr.io/kube-controller-manager:v1.17.17
docker tag registry.aliyuncs.com/google_containers/kube-scheduler:v1.17.17 k8s.gcr.io/kube-scheduler:v1.17.17
docker tag registry.aliyuncs.com/google_containers/coredns:1.6.5 k8s.gcr.io/coredns:1.6.5
docker tag registry.aliyuncs.com/google_containers/etcd:3.4.3-0 k8s.gcr.io/etcd:3.4.3-0
docker tag registry.aliyuncs.com/google_containers/pause:3.1 k8s.gcr.io/pause:3.1

cat > /etc/sysconfig/kubelet <<EOF
KUBELET_EXTRA_ARGS="--fail-swap-on=false
--runtime-cgroups=/systemd/system.slice
--kubelet-cgroups=/systemd/system.slice
--allowed-unsafe-sysctls 'kernel.msg*,net.core.somaxconn"
EOF

kubeadm init
--apiserver-advertise-address=10.3.65.37
--image-repository registry.aliyuncs.com/google_containers
--kubernetes-version v1.17.17
--service-cidr=10.96.0.0/16
--pod-network-cidr=192.168.0.0/16

mkdir -p $HOME/.kube
cp -i /etc/kubernetes/admin.conf $HOME/.kube/config

######安装flannel###########
cd ~
wget https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
kubectl apply -f kube-flannel.yml
kubectl get pod -n kube-system

######安装dashboard和Weave Scope######

kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.3.1/aio/deploy/recommended.yaml
kubectl apply -f "https://cloud.weave.works/k8s/scope.yaml?k8s-version=$(kubectl version | base64 | tr -d ' ')"

• 安装docker

访问http://mirrors.aliyun.com/docker-ce/linux/centos/，获取docker-ce.repo地址

cd /etc/yum.repos.d/
wget http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
scp到各个机器

yum -y install docker-ce

 

• 启动docker

iptables相关参考： https://blog.csdn.net/m350058411/article/details/105456953

#这一步我试了，可以不用操作，指定镜像仓库就行了
vim /lib/systemd/system/docker.service
加入以下：
ExecStartPost=/sbin/iptables -P FORWARD ACCEPT

通过默认的k8s.gcr.io镜像仓库获取kuberneters组件的相关镜像，需要配置代理

Environment="HTTPS_PROXY=http://www.ik8s.ip:10080"
Environment="NO_PROXY=10.3.0.0/16,127.0.0.0/8"

systemctl daemon-reload

systemctl restart docker

systemctl enable docker 

 

• 安装kubernetes相关组件

准备 kubeadm kubectl kubelet

 cat > /etc/yum.repos.d/kubernetes.repo <<EOF
[kubernetes]
name=Kubernetes Repo
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/
gpgcheck=1
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
　　　　https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg
enabled=1
EOF


yum repolist

yum list all | grep "^kube"

yum list kubeadm --showduplicates
yum remove kubectl kubeadm kubelet
#安装自己需要的版本
#yum -y install kubectl-1.17.3-0 kubeadm-1.17.3-0 kubelet-1.17.3-0

yum -y install kubectl kubeadm kubelet

rpm -ql kubelet

从阿里云镜像云下载kubeadm需要的镜像
kubeadm config images pull --image-repository=registry.aliyuncs.com/google_containers

#!/bin/bash

gcr_name=k8s.gcr.io
hub_name=mirrorgooglecontainers
# define images
images=(
kubernetes-dashboard-amd64:v1.10.1
kube-apiserver:v1.15.0
kube-controller-manager:v1.15.0
kube-scheduler:v1.15.0
kube-proxy:v1.15.0
pause:3.1
etcd:3.3.10
)

for image in ${images[@]}; do
        docker pull $hub_name/$image
        docker tag $hub_name/$image $gcr_name/$image
        docker rmi $hub_name/$image
done

docker pull coredns/coredns:1.3.1
docker tag coredns/coredns:1.3.1 k8s.gcr.io/coredns:1.3.1
docker rmi coredns/coredns:1.3.1

baseurl链接地址：

 gpgkey地址：

 

 

• 初始化集群

https://blog.csdn.net/Jerry_Pan1990/article/details/103233485

 vi /etc/sysconfig/kubelet  加入

KUBELET_EXTRA_ARGS="--fail-swap-on=false
--runtime-cgroups=/systemd/system.slice
--kubelet-cgroups=/systemd/system.slice
--allowed-unsafe-sysctls 'kernel.msg*,net.core.somaxconn"

vi /etc/docker/daemon.json  加入

{
"bip": "192.168.17.1/24",
"exec-opts": ["native.cgroupdriver=systemd"],
"registry-mirrors": ["https://mirror.baidubce.com", "http://docker.mirrors.ustc.edu.cn", "http://hub-mirror.c.163.com"],
"log-driver":"json-file",
"log-opts": {"max-file": "20","max-size": "100m"}
}

pod网络由网络插件指定 

flannel： 10.244.0.0/16

calico: 192.168.0.0/16

kubeadm config print init-defaults

kubeadm init --kubernetes-version v1.17.3 --pod-network-cidr 10.244.0.0/16 --ignore-preflight-errors=NumCPU

kubeadm init --pod-network-cidr 10.244.0.0/16 --ignore-preflight-errors=NumCPU

kubeadm init
--apiserver-advertise-address=10.3.65.37
--image-repository registry.aliyuncs.com/google_containers
--kubernetes-version v1.17.17
--service-cidr=10.96.0.0/16
--pod-network-cidr=192.168.0.0/16

 

 

kubeadm config images list
#查看需要哪些镜像，可以提前拉取

kubeadm config images pull  初始化前拉取镜像命令

• 初始化网络

找到flannel地址，如下图，
https://github.com/flannel-io/flannel
kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
kubectl get pod -n kube-system

 

kubeapply -f kube-flannel.yml #yml文件提前下载上传的

kubectl get pod -n kube-system

•  安装dashboard和Weave Scope

kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.3.1/aio/deploy/recommended.yaml

kubectl proxy
访问http://localhost:8001/api/v1/namespaces/kubernetes-dashboard/services/https:kubernetes-dashboard:/proxy/

kubectl apply -f "https://cloud.weave.works/k8s/scope.yaml?k8s-version=$(kubectl version | base64 | tr -d '
')"

http://localhost:4040

由于网络限制，访问不了，你们自己试试访问吧

 

kubeadm join 10.3.65.37:6443 --token e9kub4.1y1tsheeinjonyaz
--discovery-token-ca-cert-hash sha256:d11a8901ec1459035cbd9265d6421438a16af110b506929a4a504809dd0676cf

 master加入命令：

kubeadm join csapi.ejuops.com:6443 --token m1q6tq.qtahveqvo4g5w97g
--discovery-token-ca-cert-hash sha256:efe8d7fb1d51549e11baf358dad1db7e5508e3ee12123bb2c38250f0123efbe1
--control-plane --certificate-key b8022fc2f3e5561cbaeeec64005e45bcf4334c157ce3ffc9e48c8e62ba54ab76

#2、如果超过2小时忘记了令牌，可以这样做

kubeadm token create --print-join-command #打印新令牌

kubeadm token create --ttl 0 --print-join-command #创建个永不过期的令牌

 node多次kubeadm reset 又加入集群，要清理配置

kubeadm reset
rm -rf /etc/cni/net.d
systemctl stop kubelet
systemctl stop docker
rm -rf /var/lib/cni/
rm -rf /var/lib/kubelet/*
rm -rf /etc/cni/
ifconfig cni0 down
ifconfig flannel.1 down
ifconfig docker0 down
ip link delete cni0
ip link delete flannel.1

发现中文社区有个不错的文档

https://www.kubernetes.org.cn/6634.html