kubernetes集群环境搭建(3)
一、kubernetes集群环境搭建
1.1.1 部署Master节点etcd服务
- 部署etcd集群
在hdss-12 hdss-21 hdss-22上部署
| 主机名 |
角色 |
ip |
| hdss-12.hoat.com |
etcd lead |
10.0.0.12 |
| hdss-21.hoat.com |
etcd follow |
10.0.0.21 |
| hdss-22.hoat.com |
etcd follow |
10.0.0.22 |
- 在HDss-201上创建基于根证书的config配置文件
[root@hdss-201.host.com ~]# vim /opt/certs/ca-config.json
[root@hdss-201.host.com ~]# cat /opt/certs/ca-config.json
{
"signing": {
"default": {
"expiry": "175200h"
},
"profiles": {
"server": {
"expiry": "175200h",
"usages": [
"signing",
"key encipherment",
"server auth"
]
},
"client": {
"expiry": "175200h",
"usages": [
"signing",
"key encipherment",
"client auth"
]
},
"peer": {
"expiry": "175200h",
"usages": [
"signing",
"key encipherment",
"server auth",
"client auth"
]
}
}
}
}
[root@hdss-201.host.com ~]# vim /opt/certs/etcd-peer-csr.json
[root@hdss-201.host.com ~]# cat /opt/certs/etcd-peer-csr.json
{
"CN": "k8s-etcd",
"hosts": [
"10.0.0.11", #此为预留，etcd故障有这台顶替
"10.0.0.12",
"10.0.0.21",
"10.0.0.22"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "guizhou",
"L": "guiyang",
"O": "od",
"OU": "ops"
}
]
}
- 生成证书签名
[root@hdss-201.host.com ~]# cd /opt/certs/
[root@hdss-201.host.com /opt/certs]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=peer etcd-peer-csr.json |cfssl-json -bare etcd-peer
2020/06/12 21:33:29 [INFO] generate received request
2020/06/12 21:33:29 [INFO] received CSR
2020/06/12 21:33:29 [INFO] generating key: rsa-2048
2020/06/12 21:33:29 [INFO] encoded CSR
2020/06/12 21:33:29 [INFO] signed certificate with serial number 699404707025414698261653929857023788138152596776
2020/06/12 21:33:29 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").
[root@hdss-201.host.com /opt/certs]# ll
total 36
-rw-r--r-- 1 root root 840 Jun 12 21:24 ca-config.json
-rw-r--r-- 1 root root 993 Jun 10 21:49 ca.csr
-rw-r--r-- 1 root root 345 Jun 10 21:48 ca-csr.json
-rw------- 1 root root 1675 Jun 10 21:49 ca-key.pem
-rw-r--r-- 1 root root 1346 Jun 10 21:49 ca.pem
-rw-r--r-- 1 root root 1062 Jun 12 21:33 etcd-peer.csr
-rw-r--r-- 1 root root 363 Jun 12 21:27 etcd-peer-csr.json
-rw------- 1 root root 1679 Jun 12 21:33 etcd-peer-key.pem
-rw-r--r-- 1 root root 1428 Jun 12 21:33 etcd-peer.pem
- 在etcd主机上创建etcd用户
#12
[root@hdds-12.host.com ~]# useradd -M -s /sbin/nologin etcd
[root@hdds-12.host.com ~]# id etcd
uid=1001(etcd) gid=1001(etcd) groups=1001(etcd)
#21
[root@hdss-21.host.com ~]# useradd -M -s /sbin/nologin etcd
[root@hdss-21.host.com ~]# id etcd
uid=1001(etcd) gid=1001(etcd) groups=1001(etcd)
#22
[root@hdss-22.host.com ~]# useradd -M -s /sbin/nologin etcd
[root@hdss-22.host.com ~]# id etcd
uid=1001(etcd) gid=1001(etcd) groups=1001(etcd)
- 下载etcd
下载地址：https://github.com/etcd-io/etcd/releases
#12
[root@hdds-12.host.com ~]# mkdir /opt/src
[root@hdds-12.host.com ~]# cd /opt/src/
[root@hdds-12.host.com /opt/src]# rz
rz waiting to receive.
[root@hdds-12.host.com /opt/src]# ll
total 9620
-rw-r--r-- 1 root root 9850227 Apr 27 14:37 etcd-v3.1.20-linux-amd64.tar.gz
#21
[root@hdds-21.host.com ~]# mkdir /opt/src
[root@hdds-21.host.com ~]# cd /opt/src/
[root@hdds-21.host.com /opt/src]# rz
rz waiting to receive.
[root@hdds-21.host.com /opt/src]# ll
total 9620
-rw-r--r-- 1 root root 9850227 Apr 27 14:37 etcd-v3.1.20-linux-amd64.tar.gz
#22
[root@hdds-22.host.com ~]# mkdir /opt/src
[root@hdds-22.host.com ~]# cd /opt/src/
[root@hdds-22.host.com /opt/src]# rz
rz waiting to receive.
[root@hdds-22.host.com /opt/src]# ll
total 9620
-rw-r--r-- 1 root root 9850227 Apr 27 14:37 etcd-v3.1.20-linux-amd64.tar.gz
- 解压安装etcd
#12
[root@hdds-12.host.com /opt/src]# tar zxf etcd-v3.1.20-linux-amd64.tar.gz -C /opt
[root@hdds-12.host.com /opt/src]# mv /opt/etcd-v3.1.20-linux-amd64 /opt/etcd-v3.1.20
[root@hdds-12.host.com /opt/src]# ln -s /opt/etcd-v3.1.20/ /opt/etcd
[root@hdds-12.host.com /opt/src]# ll /opt/etcd
lrwxrwxrwx 1 root root 18 Jun 12 22:00 /opt/etcd -> /opt/etcd-v3.1.20/
[root@hdds-12.host.com /opt/src]# ll /opt
total 0
lrwxrwxrwx 1 root root 18 Jun 12 22:00 etcd -> /opt/etcd-v3.1.20/
drwxr-xr-x 3 478493 89939 123 Oct 11 2018 etcd-v3.1.20
drwxr-xr-x 2 root root 45 Jun 12 21:54 src
#21
[root@hdss-21.host.com /opt/src]# tar zxf etcd-v3.1.20-linux-amd64.tar.gz -C /opt
[root@hdss-21.host.com /opt/src]# mv /opt/etcd-v3.1.20-linux-amd64 /opt/etcd-v3.1.20
[root@hdss-21.host.com /opt/src]# ln -s /opt/etcd-v3.1.20/ /opt/etcd
[root@hdss-21.host.com /opt/src]# ll /opt/etcd
lrwxrwxrwx 1 root root 18 Jun 12 22:03 /opt/etcd -> /opt/etcd-v3.1.20/
[root@hdss-21.host.com /opt/src]# ll /opt/
total 0
drwx--x--x 4 root root 28 Jun 10 23:07 containerd
lrwxrwxrwx 1 root root 18 Jun 12 22:03 etcd -> /opt/etcd-v3.1.20/
drwxr-xr-x 3 478493 89939 123 Oct 11 2018 etcd-v3.1.20
drwxr-xr-x 2 root root 45 Jun 12 21:56 src
#22
[root@hdss-22.host.com /opt/src]# tar zxf etcd-v3.1.20-linux-amd64.tar.gz -C /opt
[root@hdss-22.host.com /opt/src]# mv /opt/etcd-v3.1.20-linux-amd64 /opt/etcd-v3.1.20
[root@hdss-22.host.com /opt/src]# ln -s /opt/etcd-v3.1.20/ /opt/etcd
[root@hdss-22.host.com /opt/src]# ll /opt/etcd
lrwxrwxrwx 1 root root 18 Jun 12 22:07 /opt/etcd -> /opt/etcd-v3.1.20/
[root@hdss-22.host.com /opt/src]# ll /opt/
total 0
drwx--x--x 4 root root 28 Jun 10 23:09 containerd
lrwxrwxrwx 1 root root 18 Jun 12 22:07 etcd -> /opt/etcd-v3.1.20/
drwxr-xr-x 3 478493 89939 123 Oct 11 2018 etcd-v3.1.20
drwxr-xr-x 2 root root 45 Jun 12 22:06 src
- 创建目录，拷贝证书、私钥
#12
[root@hdds-12.host.com /opt/src]# mkdir -p /opt/etcd/certs /data/etcd /data/logs/etcd-server
[root@hdds-12.host.com /opt/src]# scp -rp hdss-201:/opt/etcd/certs/ca.pem /opt/etcd/certs/
The authenticity of host 'hdss-201 (10.0.0.201)' can't be established.
ECDSA key fingerprint is SHA256:bbt9sjPOENs3zK9cw7YmIo0ABuFkZnTxXbOaIdpSOo0.
ECDSA key fingerprint is MD5:e5:3b:15:2e:6c:82:4b:b1:f8:45:dc:80:72:de:11:47.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added 'hdss-201,10.0.0.201' (ECDSA) to the list of known hosts.
root@hdss-201's password:
scp: /opt/etcd/certs/ca.pem: No such file or directory
[root@hdds-12.host.com /opt/src]# scp -rp hdss-201:/opt/certs/ca.pem /opt/etcd/certs/
root@hdss-201's password:
ca.pem 100% 1346 1.6MB/s 00:00
[root@hdds-12.host.com /opt/src]# scp -rp hdss-201:/opt/certs/etcd-peer.pem /opt/etcd/certs/
root@hdss-201's password:
etcd-peer.pem 100% 1428 1.1MB/s 00:00
[root@hdds-12.host.com /opt/src]# scp -rp hdss-201:/opt/certs/etcd-peer-key.pem /opt/etcd/certs/
root@hdss-201's password:
etcd-peer-key.pem 100% 1679 1.6MB/s 00:00
[root@hdds-12.host.com /opt/src]# ll /opt/etcd/certs/
total 12
-rw-r--r-- 1 root root 1346 Jun 10 21:49 ca.pem
-rw------- 1 root root 1679 Jun 12 21:33 etcd-peer-key.pem
-rw-r--r-- 1 root root 1428 Jun 12 21:33 etcd-peer.pem
#更改属主属组
[root@hdds-12.host.com /opt/etcd]# chown -R etcd.etcd /opt/etcd/certs /data/etcd /data/logs/etcd-server
[root@hdds-12.host.com /opt/etcd]# ll -ld /opt/etcd/certs /data/etcd /data/logs/etcd-server
drwxr-xr-x 2 etcd etcd 6 Jun 12 22:13 /data/etcd
drwxr-xr-x 2 etcd etcd 6 Jun 12 22:13 /data/logs/etcd-server
drwxr-xr-x 2 etcd etcd 66 Jun 12 22:19 /opt/etcd/certs
#21和22 同上
- 创建etcd服务启动脚本IP地址改成本机IP
[root@hdds-12.host.com /opt/src]# cd /opt/etcd
[root@hdds-12.host.com /opt/etcd]# ll
total 30068
drwxr-xr-x 2 root root 66 Jun 12 22:19 certs
drwxr-xr-x 11 478493 89939 4096 Oct 11 2018 Documentation
-rwxr-xr-x 1 478493 89939 16406432 Oct 11 2018 etcd
-rwxr-xr-x 1 478493 89939 14327712 Oct 11 2018 etcdctl
-rw-r--r-- 1 478493 89939 32632 Oct 11 2018 README-etcdctl.md
-rw-r--r-- 1 478493 89939 5878 Oct 11 2018 README.md
-rw-r--r-- 1 478493 89939 7892 Oct 11 2018 READMEv2-etcdctl.md
[root@hdds-12.host.com /opt/etcd]# ./etcd --help #etcd 配置可选参数
#12
[root@hdds-12.host.com /opt/etcd]# cat etcd-server-startup.sh
#!/bin/sh
./etcd --name etcd-server-12
--data-dir /data/etcd/etcd-server
--listen-peer-urls https://10.0.0.12:2380
--listen-client-urls https://10.0.0.12:2379,http://127.0.0.1:2379
--quota-backend-bytes 8000000000
--initial-advertise-peer-urls https://10.0.0.12:2380
--advertise-client-urls https://10.0.0.12:2379,http://127.0.0.1:2379
--initial-cluster etcd-server-12=https://10.0.0.12:2380,etcd-server-21=https://10.0.0.21:2380,etcd-server-22=https://10.0.0.22:2380
--ca-file ./certs/ca.pem #注意这里指定的目录是相对目录
--cert-file ./certs/etcd-peer.pem
--key-file ./certs/etcd-peer-key.pem
--client-cert-auth
--trusted-ca-file ./certs/ca.pem
--peer-ca-file ./certs/ca.pem
--peer-cert-file ./certs/etcd-peer.pem
--peer-key-file ./certs/etcd-peer-key.pem
--peer-client-cert-auth
--peer-trusted-ca-file ./certs/ca.pem
--log-output stdout
#赋予执行权限
[root@hdds-12.host.com /opt/etcd]# chmod +x /opt/etcd/etcd-server-startup.sh
#更改属主属组
[root@hdds-12.host.com /opt/etcd]# chown -R etcd.etcd /opt/etcd-v3.1.20/ #注意更改的是源目录不是link目录
[root@hdds-12.host.com /opt/etcd]# ll /opt/etcd-v3.1.20/
total 30072
drwxr-xr-x 2 etcd etcd 66 Jun 12 22:19 certs
drwxr-xr-x 11 etcd etcd 4096 Oct 11 2018 Documentation
-rwxr-xr-x 1 etcd etcd 16406432 Oct 11 2018 etcd
-rwxr-xr-x 1 etcd etcd 14327712 Oct 11 2018 etcdctl
-rwxr-xr-x 1 etcd etcd 977 Jun 12 22:46 etcd-server-startup.sh
-rw-r--r-- 1 etcd etcd 32632 Oct 11 2018 README-etcdctl.md
-rw-r--r-- 1 etcd etcd 5878 Oct 11 2018 README.md
-rw-r--r-- 1 etcd etcd 7892 Oct 11 2018 READMEv2-etcdctl.md
[root@hdds-12.host.com /opt/etcd]# ll -d /opt/etcd-v3.1.20/
drwxr-xr-x 4 etcd etcd 166 Jun 12 22:46 /opt/etcd-v3.1.20/
#21和22除了etcd服务启动脚本IP地址改成本机IP其他都相同
- 下载supervisor使etcd后端运行
[root@hdds-12.host.com /opt/etcd]# yum install supervisor -y
[root@hdds-12.host.com /opt/etcd]# systemctl start supervisord
[root@hdds-12.host.com /opt/etcd]# systemctl enable supervisord
Created symlink from /etc/systemd/system/multi-user.target.wants/supervisord.service to /usr/lib/systemd/system/supervisord.service.
[root@hdds-12.host.com /opt/etcd]# systemctl status supervisord
● supervisord.service - Process Monitoring and Control Daemon
Loaded: loaded (/usr/lib/systemd/system/supervisord.service; enabled; vendor preset: disabled)
Active: active (running) since Fri 2020-06-12 23:16:46 CST; 40s ago
Main PID: 6557 (supervisord)
CGroup: /system.slice/supervisord.service
└─6557 /usr/bin/python /usr/bin/supervisord -c /etc/supervisord.conf
Jun 12 23:16:46 hdds-12.host.com systemd[1]: Starting Process Monitoring and Control Daemon...
Jun 12 23:16:46 hdds-12.host.com systemd[1]: Started Process Monitoring and Control Daemon.
#创建配置管理文件
[root@hdds-12.host.com /opt/etcd]# vim /etc/supervisord.d/etcd-server.ini
[root@hdds-12.host.com /opt/etcd]# cat /etc/supervisord.d/etcd-server.ini
[program:etcd-server-12] #注意这里如果是放在21 22 上要改成21 22
command=/opt/etcd/etcd-server-startup.sh ; the program (relative uses PATH, can take args)
numprocs=1 ; number of processes copies to start (def 1)
directory=/opt/etcd ; directory to cwd to before exec (def no cwd)
autostart=true ; start at supervisord start (default: true)
autorestart=true ; retstart at unexpected quit (default: true)
startsecs=30 ; number of secs prog must stay running (def. 1)
startretries=3 ; max # of serial start failures (default 3)
exitcodes=0,2 ; 'expected' exit codes for process (default 0,2)
stopsignal=QUIT ; signal used to kill process (default TERM)
stopwaitsecs=10 ; max num secs to wait b4 SIGKILL (default 10)
user=etcd ; setuid to this UNIX account to run the program
redirect_stderr=true ; redirect proc stderr to stdout (default false)
stdout_logfile=/data/logs/etcd-server/etcd.stdout.log ; stdout log path, NONE for none; default AUTO
stdout_logfile_maxbytes=64MB ; max # logfile bytes b4 rotation (default 50MB)
stdout_logfile_backups=4 ; # of stdout logfile backups (default 10)
stdout_capture_maxbytes=1MB ; number of bytes in 'capturemode' (default 0)
stdout_events_enabled=false ; emit events on stdout writes (default false)
- 创建后端启动etcd
[root@hdds-12.host.com /opt/etcd]# supervisorctl update
etcd-server-12: added process group
[root@hdds-12.host.com /opt/etcd]# supervisorctl status
etcd-server-12 STARTING
[root@hdds-12.host.com /opt/etcd]# netstat -luntp|grep etcd
tcp 0 0 10.0.0.12:2379 0.0.0.0:* LISTEN 6590/./etcd
tcp 0 0 127.0.0.1:2379 0.0.0.0:* LISTEN 6590/./etcd
tcp 0 0 10.0.0.12:2380 0.0.0.0:* LISTEN 6590/./etcd
#21和22操作同上
- 检查集群节点
[root@hdds-12.host.com /opt/etcd]# ./etcdctl cluster-health
member 6cbdd801d2c800d9 is healthy: got healthy result from http://127.0.0.1:2379
member 74538ef5dc383e39 is healthy: got healthy result from http://127.0.0.1:2379
member f7a9c20602b8532e is healthy: got healthy result from http://127.0.0.1:2379
cluster is healthy
[root@hdds-12.host.com /opt/etcd]# ./etcdctl member list
6cbdd801d2c800d9: name=etcd-server-21 peerURLs=https://10.0.0.21:2380 clientURLs=http://127.0.0.1:2379,https://10.0.0.21:2379 isLeader=false
74538ef5dc383e39: name=etcd-server-22 peerURLs=https://10.0.0.22:2380 clientURLs=http://127.0.0.1:2379,https://10.0.0.22:2379 isLeader=true
f7a9c20602b8532e: name=etcd-server-12 peerURLs=https://10.0.0.12:2380 clientURLs=http://127.0.0.1:2379,https://10.0.0.12:2379 isLeader=false
#都是正常的
- 命令软连接
[root@hdds-12.host.com /opt/etcd]# ln -s /opt/etcd/etcdctl /usr/sbin/etcdctl
[root@hdds-12.host.com /opt/etcd]# ln -s /opt/etcd/etcdctl /usr/bin/etcdctl
[root@hdds-12.host.com /opt/etcd]# ln -s /opt/etcd/etcd /usr/bin/etcd
[root@hdds-12.host.com /opt/etcd]# ln -s /opt/etcd/etcd /usr/sbin/etcd